From a4f9debda8066219dee991b9f4465a99d4cfdfcc Mon Sep 17 00:00:00 2001 From: Jamie Atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 21 Oct 2025 13:38:59 +0200 Subject: [PATCH 01/97] Unify config files to expedition.yaml (#217) Consolidates/unifies the old dual ship_config.yaml and schedule.yaml config files into one expedition.yaml file, in line with v1 dev objectives. --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- docs/user-guide/quickstart.md | 8 +- .../tutorials/Argo_data_tutorial.ipynb | 47 +- src/virtualship/cli/_fetch.py | 28 +- src/virtualship/cli/_plan.py | 1245 ++++++++--------- src/virtualship/cli/commands.py | 35 +- src/virtualship/expedition/do_expedition.py | 64 +- .../expedition/simulate_measurements.py | 23 +- .../expedition/simulate_schedule.py | 90 +- src/virtualship/models/__init__.py | 13 +- src/virtualship/models/expedition.py | 456 ++++++ src/virtualship/models/schedule.py | 236 ---- src/virtualship/models/ship_config.py | 320 ----- src/virtualship/static/expedition.yaml | 75 + src/virtualship/static/schedule.yaml | 42 - src/virtualship/static/ship_config.yaml | 30 - src/virtualship/utils.py | 63 +- tests/cli/test_cli.py | 25 +- tests/cli/test_fetch.py | 28 +- tests/cli/test_plan.py | 30 +- .../expedition/expedition_dir/expedition.yaml | 46 + tests/expedition/expedition_dir/schedule.yaml | 18 - .../expedition_dir/ship_config.yaml | 25 - tests/expedition/test_expedition.py | 277 ++++ tests/expedition/test_schedule.py | 160 --- tests/expedition/test_ship_config.py | 126 -- tests/expedition/test_simulate_schedule.py | 31 +- tests/test_mfp_to_yaml.py | 12 +- tests/test_utils.py | 26 +- 30 files changed, 1700 insertions(+), 1883 deletions(-) create mode 100644 src/virtualship/models/expedition.py delete mode 100644 src/virtualship/models/schedule.py delete mode 100644 src/virtualship/models/ship_config.py create mode 100644 src/virtualship/static/expedition.yaml delete mode 100644 src/virtualship/static/schedule.yaml delete mode 100644 src/virtualship/static/ship_config.yaml create mode 100644 tests/expedition/expedition_dir/expedition.yaml delete mode 100644 tests/expedition/expedition_dir/schedule.yaml delete mode 100644 tests/expedition/expedition_dir/ship_config.yaml create mode 100644 tests/expedition/test_expedition.py delete mode 100644 tests/expedition/test_schedule.py delete mode 100644 tests/expedition/test_ship_config.py diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 138cbb37..b5373470 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,7 +1,7 @@ --- name: Bug report about: Create a report to help us improve -title: "" +title: ["needs-triage"] labels: bug assignees: "" --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 8e47557f..1adc441f 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,7 +1,7 @@ --- name: Feature request about: Suggest an idea for this project -title: "" +title: ["needs-triage"] labels: enhancement assignees: "" --- diff --git a/docs/user-guide/quickstart.md b/docs/user-guide/quickstart.md index 59a514c7..45d4050f 100644 --- a/docs/user-guide/quickstart.md +++ b/docs/user-guide/quickstart.md @@ -46,10 +46,10 @@ virtualship init EXPEDITION_NAME --from-mfp CoordinatesExport.xlsx The `CoordinatesExport.xlsx` in the `virtualship init` command refers to the .xlsx file exported from MFP. Replace the filename with the name of your exported .xlsx file (and make sure to move it from the Downloads to the folder/directory in which you are running the expedition). ``` -This will create a folder/directory called `EXPEDITION_NAME` with two files: `schedule.yaml` and `ship_config.yaml` based on the sampling site coordinates that you specified in your MFP export. The `--from-mfp` flag indictates that the exported coordinates will be used. +This will create a folder/directory called `EXPEDITION_NAME` with a single file: `expedition.yaml` containing details on the ship and instrument configurations, as well as the expedition schedule based on the sampling site coordinates that you specified in your MFP export. The `--from-mfp` flag indicates that the exported coordinates will be used. ```{note} -For advanced users: it is also possible to run the expedition initialisation step without an MFP .xlsx export file. In this case you should simply run `virtualship init EXPEDITION_NAME` in the CLI. This will write example `schedule.yaml` and `ship_config.yaml` files in the `EXPEDITION_NAME` folder/directory. These files contain example waypoints, timings and instrument selections, but can be edited or propagated through the rest of the workflow unedited to run a sample expedition. +For advanced users: it is also possible to run the expedition initialisation step without an MFP .xlsx export file. In this case you should simply run `virtualship init EXPEDITION_NAME` in the CLI. This will write an example `expedition.yaml` file in the `EXPEDITION_NAME` folder/directory. This file contains example waypoints, timings, instrument selections, and ship configuration, but can be edited or propagated through the rest of the workflow unedited to run a sample expedition. ``` ## Expedition scheduling & ship configuration @@ -61,7 +61,7 @@ virtualship plan EXPEDITION_NAME ``` ```{tip} -Using the `virtualship plan` tool is optional. Advanced users can also edit the `schedule.yaml` and `ship_config.yaml` files directly if preferred. +Using the `virtualship plan` tool is optional. Advanced users can also edit the `expedition.yaml` file directly if preferred. ``` The planning tool should look something like this and offers an intuitive way to make your selections: @@ -111,7 +111,7 @@ For advanced users: you can also make further customisations to behaviours of al When you are happy with your ship configuration and schedule plan, press _Save Changes_. ```{note} -On pressing _Save Changes_ the tool will check the selections are valid (for example that the ship will be able to reach each waypoint in time). If they are, the changes will be saved to the `ship_config.yaml` and `schedule.yaml` files, ready for the next steps. If your selections are invalid you should be provided with information on how to fix them. +On pressing _Save Changes_ the tool will check the selections are valid (for example that the ship will be able to reach each waypoint in time). If they are, the changes will be saved to the `expedition.yaml` file, ready for the next steps. If your selections are invalid you should be provided with information on how to fix them. ``` ## Fetch the data diff --git a/docs/user-guide/tutorials/Argo_data_tutorial.ipynb b/docs/user-guide/tutorials/Argo_data_tutorial.ipynb index 30cee460..e8235315 100644 --- a/docs/user-guide/tutorials/Argo_data_tutorial.ipynb +++ b/docs/user-guide/tutorials/Argo_data_tutorial.ipynb @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -28,25 +28,26 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We have downloaded the data from Copernicus Marine Service, using `virtualship fetch` and the information in following `schedule.yaml` file:\n", + "We have downloaded the data from Copernicus Marine Service, using `virtualship fetch` and the information in following `schedule` section of the `expedition.yaml` file:\n", "```yaml\n", - "space_time_region:\n", - " spatial_range:\n", - " minimum_longitude: -5\n", - " maximum_longitude: 5\n", - " minimum_latitude: -5\n", - " maximum_latitude: 5\n", - " minimum_depth: 0\n", - " maximum_depth: 2000\n", - " time_range:\n", - " start_time: 2023-01-01 00:00:00\n", - " end_time: 2023-02-01 00:00:00\n", - "waypoints:\n", - " - instrument: ARGO_FLOAT\n", - " location:\n", - " latitude: 0.02\n", - " longitude: 0.02\n", - " time: 2023-01-01 02:00:00\n", + "schedule:\n", + " space_time_region:\n", + " spatial_range:\n", + " minimum_longitude: -5\n", + " maximum_longitude: 5\n", + " minimum_latitude: -5\n", + " maximum_latitude: 5\n", + " minimum_depth: 0\n", + " maximum_depth: 2000\n", + " time_range:\n", + " start_time: 2023-01-01 00:00:00\n", + " end_time: 2023-02-01 00:00:00\n", + " waypoints:\n", + " - instrument: ARGO_FLOAT\n", + " location:\n", + " latitude: 0.02\n", + " longitude: 0.02\n", + " time: 2023-01-01 02:00:00\n", "```\n", "\n", "After running `virtualship run`, we have a `results/argo_floats.zarr` file with the data from the float." @@ -54,7 +55,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -79,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -111,7 +112,7 @@ ], "metadata": { "kernelspec": { - "display_name": "parcels", + "display_name": "ship", "language": "python", "name": "python3" }, @@ -125,7 +126,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.9" } }, "nbformat": 4, diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index ac039d76..60008304 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -12,8 +12,7 @@ from virtualship.utils import ( _dump_yaml, _generic_load_yaml, - _get_schedule, - _get_ship_config, + _get_expedition, ) if TYPE_CHECKING: @@ -24,7 +23,7 @@ from copernicusmarine.core_functions.credentials_utils import InvalidUsernameOrPassword import virtualship.cli._creds as creds -from virtualship.utils import SCHEDULE +from virtualship.utils import EXPEDITION DOWNLOAD_METADATA = "download_metadata.yaml" @@ -49,17 +48,18 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None data_folder = path / "data" data_folder.mkdir(exist_ok=True) - schedule = _get_schedule(path) - ship_config = _get_ship_config(path) + expedition = _get_expedition(path) - schedule.verify( - ship_config.ship_speed_knots, + expedition.schedule.verify( + expedition.ship_config.ship_speed_knots, input_data=None, check_space_time_region=True, ignore_missing_fieldsets=True, ) - space_time_region_hash = get_space_time_region_hash(schedule.space_time_region) + space_time_region_hash = get_space_time_region_hash( + expedition.schedule.space_time_region + ) existing_download = get_existing_download(data_folder, space_time_region_hash) if existing_download is not None: @@ -72,11 +72,11 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None username, password = creds.get_credentials_flow(username, password, creds_path) # Extract space_time_region details from the schedule - spatial_range = schedule.space_time_region.spatial_range - time_range = schedule.space_time_region.time_range + spatial_range = expedition.schedule.space_time_region.spatial_range + time_range = expedition.schedule.space_time_region.time_range start_datetime = time_range.start_time end_datetime = time_range.end_time - instruments_in_schedule = schedule.get_instruments() + instruments_in_schedule = expedition.schedule.get_instruments() # Create download folder and set download metadata download_folder = data_folder / hash_to_filename(space_time_region_hash) @@ -84,15 +84,15 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None DownloadMetadata(download_complete=False).to_yaml( download_folder / DOWNLOAD_METADATA ) - shutil.copyfile(path / SCHEDULE, download_folder / SCHEDULE) + shutil.copyfile(path / EXPEDITION, download_folder / EXPEDITION) if ( ( {"XBT", "CTD", "CDT_BGC", "SHIP_UNDERWATER_ST"} & set(instrument.name for instrument in instruments_in_schedule) ) - or ship_config.ship_underwater_st_config is not None - or ship_config.adcp_config is not None + or expedition.instruments_config.ship_underwater_st_config is not None + or expedition.instruments_config.adcp_config is not None ): print("Ship data will be downloaded. Please wait...") diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 85539e3f..87bfe336 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -1,7 +1,6 @@ import datetime import os import traceback -from typing import ClassVar from textual import on from textual.app import App, ComposeResult @@ -30,23 +29,23 @@ type_to_textual, ) from virtualship.errors import UnexpectedError, UserError -from virtualship.models.location import Location -from virtualship.models.schedule import Schedule, Waypoint -from virtualship.models.ship_config import ( +from virtualship.models import ( ADCPConfig, ArgoFloatConfig, CTD_BGCConfig, CTDConfig, DrifterConfig, + Expedition, InstrumentType, + Location, ShipConfig, ShipUnderwaterSTConfig, - XBTConfig, -) -from virtualship.models.space_time_region import ( SpatialRange, TimeRange, + Waypoint, + XBTConfig, ) +from virtualship.utils import EXPEDITION UNEXPECTED_MSG_ONSAVE = ( "Please ensure that:\n" @@ -81,227 +80,236 @@ def log_exception_to_file( f.write("\n") -class WaypointWidget(Static): - def __init__(self, waypoint: Waypoint, index: int): +DEFAULT_TS_CONFIG = {"period_minutes": 5.0} + +DEFAULT_ADCP_CONFIG = { + "num_bins": 40, + "period_minutes": 5.0, +} + +INSTRUMENT_FIELDS = { + "adcp_config": { + "class": ADCPConfig, + "title": "Onboard ADCP", + "attributes": [ + {"name": "num_bins"}, + {"name": "period", "minutes": True}, + ], + }, + "ship_underwater_st_config": { + "class": ShipUnderwaterSTConfig, + "title": "Onboard Temperature/Salinity", + "attributes": [ + {"name": "period", "minutes": True}, + ], + }, + "ctd_config": { + "class": CTDConfig, + "title": "CTD", + "attributes": [ + {"name": "max_depth_meter"}, + {"name": "min_depth_meter"}, + {"name": "stationkeeping_time", "minutes": True}, + ], + }, + "ctd_bgc_config": { + "class": CTD_BGCConfig, + "title": "CTD-BGC", + "attributes": [ + {"name": "max_depth_meter"}, + {"name": "min_depth_meter"}, + {"name": "stationkeeping_time", "minutes": True}, + ], + }, + "xbt_config": { + "class": XBTConfig, + "title": "XBT", + "attributes": [ + {"name": "min_depth_meter"}, + {"name": "max_depth_meter"}, + {"name": "fall_speed_meter_per_second"}, + {"name": "deceleration_coefficient"}, + ], + }, + "argo_float_config": { + "class": ArgoFloatConfig, + "title": "Argo Float", + "attributes": [ + {"name": "min_depth_meter"}, + {"name": "max_depth_meter"}, + {"name": "drift_depth_meter"}, + {"name": "vertical_speed_meter_per_second"}, + {"name": "cycle_days"}, + {"name": "drift_days"}, + ], + }, + "drifter_config": { + "class": DrifterConfig, + "title": "Drifter", + "attributes": [ + {"name": "depth_meter"}, + {"name": "lifetime", "minutes": True}, + ], + }, +} + + +class ExpeditionEditor(Static): + def __init__(self, path: str): super().__init__() - self.waypoint = waypoint - self.index = index + self.path = path + self.expedition = None def compose(self) -> ComposeResult: try: - with Collapsible( - title=f"[b]Waypoint {self.index + 1}[/b]", - collapsed=True, - id=f"wp{self.index + 1}", - ): - if self.index > 0: - yield Button( - "Copy Time & Instruments from Previous", - id=f"wp{self.index}_copy", - variant="warning", - ) - yield Label("Location:") - yield Label(" Latitude:") - yield Input( - id=f"wp{self.index}_lat", - value=str(self.waypoint.location.lat) - if self.waypoint.location.lat - is not None # is not None to handle if lat is 0.0 - else "", - validators=[ - Function( - is_valid_lat, - f"INVALID: value must be {is_valid_lat.__doc__.lower()}", - ) - ], - type="number", - placeholder="°N", - classes="latitude-input", - ) - yield Label( - "", - id=f"validation-failure-label-wp{self.index}_lat", - classes="-hidden validation-failure", - ) + self.expedition = Expedition.from_yaml(self.path.joinpath(EXPEDITION)) + except Exception as e: + raise UserError( + f"There is an issue in {self.path.joinpath(EXPEDITION)}:\n\n{e}" + ) from None - yield Label(" Longitude:") - yield Input( - id=f"wp{self.index}_lon", - value=str(self.waypoint.location.lon) - if self.waypoint.location.lon - is not None # is not None to handle if lon is 0.0 - else "", - validators=[ - Function( - is_valid_lon, - f"INVALID: value must be {is_valid_lon.__doc__.lower()}", - ) - ], - type="number", - placeholder="°E", - classes="longitude-input", - ) - yield Label( - "", - id=f"validation-failure-label-wp{self.index}_lon", - classes="-hidden validation-failure", - ) + try: + ## 1) SHIP SPEED & INSTRUMENTS CONFIG EDITOR - yield Label("Time:") - with Horizontal(): - yield Label("Year:") - yield Select( - [ - (str(year), year) - # TODO: change from hard coding? ...flexibility for different datasets... - for year in range( - 2022, - datetime.datetime.now().year + 1, + yield Label( + "[b]Ship & Instruments Config Editor[/b]", + id="title_ship_instruments_config", + markup=True, + ) + yield Rule(line_style="heavy") + + # SECTION: "Ship Speed & Onboard Measurements" + + with Collapsible( + title="[b]Ship Speed & Onboard Measurements[/b]", + id="speed_collapsible", + collapsed=False, + ): + attr = "ship_speed_knots" + validators = group_validators(ShipConfig, attr) + with Horizontal(classes="ship_speed"): + yield Label("[b]Ship Speed (knots):[/b]") + yield Input( + id="speed", + type=type_to_textual(get_field_type(ShipConfig, attr)), + validators=[ + Function( + validator, + f"INVALID: value must be {validator.__doc__.lower()}", ) + for validator in validators ], - id=f"wp{self.index}_year", - value=int(self.waypoint.time.year) - if self.waypoint.time - else Select.BLANK, - prompt="YYYY", - classes="year-select", - ) - yield Label("Month:") - yield Select( - [(f"{m:02d}", m) for m in range(1, 13)], - id=f"wp{self.index}_month", - value=int(self.waypoint.time.month) - if self.waypoint.time - else Select.BLANK, - prompt="MM", - classes="month-select", + classes="ship_speed_input", + placeholder="knots", + value=str( + self.expedition.ship_config.ship_speed_knots + if self.expedition.ship_config.ship_speed_knots + else "" + ), ) - yield Label("Day:") - yield Select( - [(f"{d:02d}", d) for d in range(1, 32)], - id=f"wp{self.index}_day", - value=int(self.waypoint.time.day) - if self.waypoint.time - else Select.BLANK, - prompt="DD", - classes="day-select", + yield Label("", id="validation-failure-label-speed", classes="-hidden") + + with Horizontal(classes="ts-section"): + yield Label("[b]Onboard Temperature/Salinity:[/b]") + yield Switch( + value=bool( + self.expedition.instruments_config.ship_underwater_st_config + ), + id="has_onboard_ts", ) - yield Label("Hour:") - yield Select( - [(f"{h:02d}", h) for h in range(24)], - id=f"wp{self.index}_hour", - value=int(self.waypoint.time.hour) - if self.waypoint.time - else Select.BLANK, - prompt="hh", - classes="hour-select", + + with Horizontal(classes="adcp-section"): + yield Label("[b]Onboard ADCP:[/b]") + yield Switch( + value=bool(self.expedition.instruments_config.adcp_config), + id="has_adcp", ) - yield Label("Min:") - yield Select( - [(f"{m:02d}", m) for m in range(0, 60, 5)], - id=f"wp{self.index}_minute", - value=int(self.waypoint.time.minute) - if self.waypoint.time - else Select.BLANK, - prompt="mm", - classes="minute-select", + + # adcp type selection + with Horizontal(id="adcp_type_container", classes="-hidden"): + is_deep = ( + self.expedition.instruments_config.adcp_config + and self.expedition.instruments_config.adcp_config.max_depth_meter + == -1000.0 ) + yield Label(" OceanObserver:") + yield Switch(value=is_deep, id="adcp_deep") + yield Label(" SeaSeven:") + yield Switch(value=not is_deep, id="adcp_shallow") + yield Button("?", id="info_button", variant="warning") - yield Label("Instruments:") - for instrument in InstrumentType: - is_selected = instrument in (self.waypoint.instrument or []) - with Horizontal(): - yield Label(instrument.value) - yield Switch( - value=is_selected, id=f"wp{self.index}_{instrument.value}" - ) + ## SECTION: "Instrument Configurations"" - if instrument.value == "DRIFTER": - yield Label("Count") - yield Input( - id=f"wp{self.index}_drifter_count", - value=str( - self.get_drifter_count() if is_selected else "" - ), - type="integer", - placeholder="# of drifters", - validators=Integer( - minimum=1, - failure_description="INVALID: value must be > 0", - ), - classes="drifter-count-input", - ) + with Collapsible( + title="[b]Instrument Configurations[/b] (advanced users only)", + collapsed=True, + ): + for instrument_name, info in INSTRUMENT_FIELDS.items(): + config_class = info["class"] + attributes = info["attributes"] + # instrument-specific configs now live under instruments_config + config_instance = getattr( + self.expedition.instruments_config, instrument_name, None + ) + title = info.get("title", instrument_name.replace("_", " ").title()) + with Collapsible( + title=f"[b]{title}[/b]", + collapsed=True, + ): + if instrument_name in ( + "adcp_config", + "ship_underwater_st_config", + ): yield Label( - "", - id=f"validation-failure-label-wp{self.index}_drifter_count", - classes="-hidden validation-failure", + f"NOTE: entries will be ignored here if {info['title']} is OFF in Ship Speed & Onboard Measurements." ) + with Container(classes="instrument-config"): + for attr_meta in attributes: + attr = attr_meta["name"] + is_minutes = attr_meta.get("minutes", False) + validators = group_validators(config_class, attr) + if config_instance: + raw_value = getattr(config_instance, attr, "") + if is_minutes and raw_value != "": + try: + value = str( + raw_value.total_seconds() / 60.0 + ) + except AttributeError: + value = str(raw_value) + else: + value = str(raw_value) + else: + value = "" + label = f"{attr.replace('_', ' ').title()}:" + yield Label( + label + if not is_minutes + else label.replace(":", " Minutes:") + ) + yield Input( + id=f"{instrument_name}_{attr}", + type=type_to_textual( + get_field_type(config_class, attr) + ), + validators=[ + Function( + validator, + f"INVALID: value must be {validator.__doc__.lower()}", + ) + for validator in validators + ], + value=value, + ) + yield Label( + "", + id=f"validation-failure-label-{instrument_name}_{attr}", + classes="-hidden validation-failure", + ) - except Exception as e: - raise UnexpectedError(unexpected_msg_compose(e)) from None - - def get_drifter_count(self) -> int: - return sum( - 1 for inst in self.waypoint.instrument if inst == InstrumentType.DRIFTER - ) - - def copy_from_previous(self) -> None: - """Copy inputs from previous waypoint widget (time and instruments only, not lat/lon).""" - try: - if self.index > 0: - schedule_editor = self.parent - if schedule_editor: - time_components = ["year", "month", "day", "hour", "minute"] - for comp in time_components: - prev = schedule_editor.query_one(f"#wp{self.index - 1}_{comp}") - curr = self.query_one(f"#wp{self.index}_{comp}") - if prev and curr: - curr.value = prev.value - - for instrument in InstrumentType: - prev_switch = schedule_editor.query_one( - f"#wp{self.index - 1}_{instrument.value}" - ) - curr_switch = self.query_one( - f"#wp{self.index}_{instrument.value}" - ) - if prev_switch and curr_switch: - curr_switch.value = prev_switch.value - except Exception as e: - raise UnexpectedError(unexpected_msg_compose(e)) from None - - @on(Button.Pressed, "Button") - def button_pressed(self, event: Button.Pressed) -> None: - if event.button.id == f"wp{self.index}_copy": - self.copy_from_previous() - - @on(Switch.Changed) - def on_switch_changed(self, event: Switch.Changed) -> None: - if event.switch.id == f"wp{self.index}_DRIFTER": - drifter_count_input = self.query_one( - f"#wp{self.index}_drifter_count", Input - ) - if not event.value: - drifter_count_input.value = "" - else: - if not drifter_count_input.value: - drifter_count_input.value = "1" - - -class ScheduleEditor(Static): - def __init__(self, path: str): - super().__init__() - self.path = path - self.schedule = None - - def compose(self) -> ComposeResult: - try: - self.schedule = Schedule.from_yaml(f"{self.path}/schedule.yaml") - except Exception as e: - raise UserError(f"There is an issue in schedule.yaml:\n\n{e}") from None + ## 2) SCHEDULE EDITOR - try: - yield Label("[b]Schedule Editor[/b]", id="title", markup=True) + yield Label("[b]Schedule Editor[/b]", id="title_schedule", markup=True) yield Rule(line_style="heavy") # SECTION: "Waypoints & Instrument Selection" @@ -327,8 +335,8 @@ def compose(self) -> ComposeResult: title="[b]Space-Time Region[/b] (advanced users only)", collapsed=True, ): - if self.schedule.space_time_region: - str_data = self.schedule.space_time_region + if self.expedition.schedule.space_time_region: + str_data = self.expedition.schedule.space_time_region yield Label("Minimum Latitude:") yield Input( @@ -501,13 +509,137 @@ def compose(self) -> ComposeResult: def on_mount(self) -> None: self.refresh_waypoint_widgets() + adcp_present = ( + getattr(self.expedition.instruments_config, "adcp_config", None) + if self.expedition.instruments_config + else False + ) + self.show_hide_adcp_type(bool(adcp_present)) def refresh_waypoint_widgets(self): waypoint_list = self.query_one("#waypoint_list", VerticalScroll) waypoint_list.remove_children() - for i, waypoint in enumerate(self.schedule.waypoints): + for i, waypoint in enumerate(self.expedition.schedule.waypoints): waypoint_list.mount(WaypointWidget(waypoint, i)) + def save_changes(self) -> bool: + """Save changes to expedition.yaml.""" + try: + self._update_ship_speed() + self._update_instrument_configs() + self._update_schedule() + self.expedition.to_yaml(self.path.joinpath(EXPEDITION)) + return True + except Exception as e: + log_exception_to_file( + e, + self.path, + context_message=f"Error saving {self.path.joinpath(EXPEDITION)}:", + ) + raise UnexpectedError( + UNEXPECTED_MSG_ONSAVE + + f"\n\nTraceback will be logged in {self.path}/virtualship_error.txt. Please attach this/copy the contents to any issue submitted." + ) from None + + def _update_ship_speed(self): + attr = "ship_speed_knots" + field_type = get_field_type(type(self.expedition.ship_config), attr) + value = field_type(self.query_one("#speed").value) + ShipConfig.model_validate( + {**self.expedition.ship_config.model_dump(), attr: value} + ) + self.expedition.ship_config.ship_speed_knots = value + + def _update_instrument_configs(self): + for instrument_name, info in INSTRUMENT_FIELDS.items(): + config_class = info["class"] + attributes = info["attributes"] + kwargs = {} + # special handling for onboard ADCP and T/S + if instrument_name == "adcp_config": + has_adcp = self.query_one("#has_adcp", Switch).value + if not has_adcp: + setattr(self.expedition.instruments_config, instrument_name, None) + continue + if instrument_name == "ship_underwater_st_config": + has_ts = self.query_one("#has_onboard_ts", Switch).value + if not has_ts: + setattr(self.expedition.instruments_config, instrument_name, None) + continue + for attr_meta in attributes: + attr = attr_meta["name"] + is_minutes = attr_meta.get("minutes", False) + input_id = f"{instrument_name}_{attr}" + value = self.query_one(f"#{input_id}").value + field_type = get_field_type(config_class, attr) + if is_minutes and field_type is datetime.timedelta: + value = datetime.timedelta(minutes=float(value)) + else: + value = field_type(value) + kwargs[attr] = value + # ADCP max_depth_meter based on deep/shallow switch + if instrument_name == "adcp_config": + if self.query_one("#adcp_deep", Switch).value: + kwargs["max_depth_meter"] = -1000.0 + else: + kwargs["max_depth_meter"] = -150.0 + setattr( + self.expedition.instruments_config, + instrument_name, + config_class(**kwargs), + ) + + def _update_schedule(self): + spatial_range = SpatialRange( + minimum_longitude=self.query_one("#min_lon").value, + maximum_longitude=self.query_one("#max_lon").value, + minimum_latitude=self.query_one("#min_lat").value, + maximum_latitude=self.query_one("#max_lat").value, + minimum_depth=self.query_one("#min_depth").value, + maximum_depth=self.query_one("#max_depth").value, + ) + start_time_input = self.query_one("#start_time").value + end_time_input = self.query_one("#end_time").value + waypoint_times = [ + wp.time + for wp in self.expedition.schedule.waypoints + if hasattr(wp, "time") and wp.time + ] + if not start_time_input and waypoint_times: + start_time = min(waypoint_times) + else: + start_time = start_time_input + if not end_time_input and waypoint_times: + end_time = max(waypoint_times) + datetime.timedelta(minutes=60480.0) + else: + end_time = end_time_input + time_range = TimeRange(start_time=start_time, end_time=end_time) + self.expedition.schedule.space_time_region.spatial_range = spatial_range + self.expedition.schedule.space_time_region.time_range = time_range + for i, wp in enumerate(self.expedition.schedule.waypoints): + wp.location = Location( + latitude=float(self.query_one(f"#wp{i}_lat").value), + longitude=float(self.query_one(f"#wp{i}_lon").value), + ) + wp.time = datetime.datetime( + int(self.query_one(f"#wp{i}_year").value), + int(self.query_one(f"#wp{i}_month").value), + int(self.query_one(f"#wp{i}_day").value), + int(self.query_one(f"#wp{i}_hour").value), + int(self.query_one(f"#wp{i}_minute").value), + 0, + ) + wp.instrument = [] + for instrument in InstrumentType: + switch_on = self.query_one(f"#wp{i}_{instrument.value}").value + if instrument.value == "DRIFTER" and switch_on: + count_str = self.query_one(f"#wp{i}_drifter_count").value + count = int(count_str) + assert count > 0 + wp.instrument.extend([InstrumentType.DRIFTER] * count) + elif switch_on: + wp.instrument.append(instrument) + @on(Input.Changed) def show_invalid_reasons(self, event: Input.Changed) -> None: input_id = event.input.id @@ -547,8 +679,8 @@ def show_invalid_reasons(self, event: Input.Changed) -> None: def add_waypoint(self) -> None: """Add a new waypoint to the schedule. Copies time from last waypoint if possible (Lat/lon and instruments blank).""" try: - if self.schedule.waypoints: - last_wp = self.schedule.waypoints[-1] + if self.expedition.schedule.waypoints: + last_wp = self.expedition.schedule.waypoints[-1] new_time = last_wp.time if last_wp.time else None new_wp = Waypoint( location=Location( @@ -558,320 +690,27 @@ def add_waypoint(self) -> None: time=new_time, instrument=[], ) - else: - new_wp = Waypoint( - location=Location(latitude=0.0, longitude=0.0), - time=None, - instrument=[], - ) - self.schedule.waypoints.append(new_wp) - self.refresh_waypoint_widgets() - - except Exception as e: - raise UnexpectedError(unexpected_msg_compose(e)) from None - - @on(Button.Pressed, "#remove_waypoint") - def remove_waypoint(self) -> None: - """Remove the last waypoint from the schedule.""" - try: - if self.schedule.waypoints: - self.schedule.waypoints.pop() - self.refresh_waypoint_widgets() - else: - self.notify("No waypoints to remove.", severity="error", timeout=5) - - except Exception as e: - raise UnexpectedError(unexpected_msg_compose(e)) from None - - def save_changes(self) -> bool: - """Save changes to schedule.yaml.""" - try: - ## spacetime region - spatial_range = SpatialRange( - minimum_longitude=self.query_one("#min_lon").value, - maximum_longitude=self.query_one("#max_lon").value, - minimum_latitude=self.query_one("#min_lat").value, - maximum_latitude=self.query_one("#max_lat").value, - minimum_depth=self.query_one("#min_depth").value, - maximum_depth=self.query_one("#max_depth").value, - ) - - # auto fill start and end times if input is blank - start_time_input = self.query_one("#start_time").value - end_time_input = self.query_one("#end_time").value - waypoint_times = [ - wp.time - for wp in self.schedule.waypoints - if hasattr(wp, "time") and wp.time - ] - - if not start_time_input and waypoint_times: - start_time = min(waypoint_times) - else: - start_time = start_time_input - - if not end_time_input and waypoint_times: - end_time = max(waypoint_times) + datetime.timedelta( - minutes=60480.0 - ) # with buffer (corresponds to default drifter lifetime) - else: - end_time = end_time_input - - time_range = TimeRange( - start_time=start_time, - end_time=end_time, - ) - - self.schedule.space_time_region.spatial_range = spatial_range - self.schedule.space_time_region.time_range = time_range - - ## waypoints - for i, wp in enumerate(self.schedule.waypoints): - wp.location = Location( - latitude=float(self.query_one(f"#wp{i}_lat").value), - longitude=float(self.query_one(f"#wp{i}_lon").value), - ) - wp.time = datetime.datetime( - int(self.query_one(f"#wp{i}_year").value), - int(self.query_one(f"#wp{i}_month").value), - int(self.query_one(f"#wp{i}_day").value), - int(self.query_one(f"#wp{i}_hour").value), - int(self.query_one(f"#wp{i}_minute").value), - 0, - ) - - wp.instrument = [] - for instrument in InstrumentType: - switch_on = self.query_one(f"#wp{i}_{instrument.value}").value - if instrument.value == "DRIFTER" and switch_on: - count_str = self.query_one(f"#wp{i}_drifter_count").value - count = int(count_str) - assert count > 0 - wp.instrument.extend([InstrumentType.DRIFTER] * count) - elif switch_on: - wp.instrument.append(instrument) - - # save - self.schedule.to_yaml(f"{self.path}/schedule.yaml") - return True - - except Exception as e: - log_exception_to_file( - e, self.path, context_message="Error saving schedule:" - ) - - raise UnexpectedError( - UNEXPECTED_MSG_ONSAVE - + f"\n\nTraceback will be logged in {self.path}/virtualship_error.txt. Please attach this/copy the contents to any issue submitted." - ) from None - - -class ConfigEditor(Container): - DEFAULT_ADCP_CONFIG: ClassVar[dict[str, float]] = { - "num_bins": 40, - "period_minutes": 5.0, - } - - DEFAULT_TS_CONFIG: ClassVar[dict[str, float]] = {"period_minutes": 5.0} - - INSTRUMENT_FIELDS: ClassVar[dict[str, dict]] = { - "adcp_config": { - "class": ADCPConfig, - "title": "Onboard ADCP", - "attributes": [ - {"name": "num_bins"}, - {"name": "period", "minutes": True}, - ], - }, - "ship_underwater_st_config": { - "class": ShipUnderwaterSTConfig, - "title": "Onboard Temperature/Salinity", - "attributes": [ - {"name": "period", "minutes": True}, - ], - }, - "ctd_config": { - "class": CTDConfig, - "title": "CTD", - "attributes": [ - {"name": "max_depth_meter"}, - {"name": "min_depth_meter"}, - {"name": "stationkeeping_time", "minutes": True}, - ], - }, - "ctd_bgc_config": { - "class": CTD_BGCConfig, - "title": "CTD-BGC", - "attributes": [ - {"name": "max_depth_meter"}, - {"name": "min_depth_meter"}, - {"name": "stationkeeping_time", "minutes": True}, - ], - }, - "xbt_config": { - "class": XBTConfig, - "title": "XBT", - "attributes": [ - {"name": "min_depth_meter"}, - {"name": "max_depth_meter"}, - {"name": "fall_speed_meter_per_second"}, - {"name": "deceleration_coefficient"}, - ], - }, - "argo_float_config": { - "class": ArgoFloatConfig, - "title": "Argo Float", - "attributes": [ - {"name": "min_depth_meter"}, - {"name": "max_depth_meter"}, - {"name": "drift_depth_meter"}, - {"name": "vertical_speed_meter_per_second"}, - {"name": "cycle_days"}, - {"name": "drift_days"}, - ], - }, - "drifter_config": { - "class": DrifterConfig, - "title": "Drifter", - "attributes": [ - {"name": "depth_meter"}, - {"name": "lifetime", "minutes": True}, - ], - }, - } - - def __init__(self, path: str): - super().__init__() - self.path = path - self.config = None - - def compose(self) -> ComposeResult: - try: - self.config = ShipConfig.from_yaml(f"{self.path}/ship_config.yaml") - except Exception as e: - raise UserError(f"There is an issue in ship_config.yaml:\n\n{e}") from None - - try: - ## SECTION: "Ship Speed & Onboard Measurements" - - yield Label("[b]Ship Config Editor[/b]", id="title", markup=True) - yield Rule(line_style="heavy") - - with Collapsible( - title="[b]Ship Speed & Onboard Measurements[/b]", id="speed_collapsible" - ): - attr = "ship_speed_knots" - validators = group_validators(ShipConfig, attr) - with Horizontal(classes="ship_speed"): - yield Label("[b]Ship Speed (knots):[/b]") - yield Input( - id="speed", - type=type_to_textual(get_field_type(ShipConfig, attr)), - validators=[ - Function( - validator, - f"INVALID: value must be {validator.__doc__.lower()}", - ) - for validator in validators - ], - classes="ship_speed_input", - placeholder="knots", - value=str( - self.config.ship_speed_knots - if self.config.ship_speed_knots - else "" - ), - ) - yield Label("", id="validation-failure-label-speed", classes="-hidden") - - with Horizontal(classes="ts-section"): - yield Label("[b]Onboard Temperature/Salinity:[/b]") - yield Switch( - value=bool(self.config.ship_underwater_st_config), - id="has_onboard_ts", - ) - - with Horizontal(classes="adcp-section"): - yield Label("[b]Onboard ADCP:[/b]") - yield Switch(value=bool(self.config.adcp_config), id="has_adcp") - - # adcp type selection - with Horizontal(id="adcp_type_container", classes="-hidden"): - is_deep = ( - self.config.adcp_config - and self.config.adcp_config.max_depth_meter == -1000.0 - ) - yield Label(" OceanObserver:") - yield Switch(value=is_deep, id="adcp_deep") - yield Label(" SeaSeven:") - yield Switch(value=not is_deep, id="adcp_shallow") - yield Button("?", id="info_button", variant="warning") - - ## SECTION: "Instrument Configurations"" - - with Collapsible( - title="[b]Instrument Configurations[/b] (advanced users only)", - collapsed=True, - ): - for instrument_name, info in self.INSTRUMENT_FIELDS.items(): - config_class = info["class"] - attributes = info["attributes"] - config_instance = getattr(self.config, instrument_name, None) - title = info.get("title", instrument_name.replace("_", " ").title()) - with Collapsible( - title=f"[b]{title}[/b]", - collapsed=True, - ): - if instrument_name in ( - "adcp_config", - "ship_underwater_st_config", - ): - yield Label( - f"NOTE: entries will be ignored here if {info['title']} is OFF in Ship Speed & Onboard Measurements." - ) - with Container(classes="instrument-config"): - for attr_meta in attributes: - attr = attr_meta["name"] - is_minutes = attr_meta.get("minutes", False) - validators = group_validators(config_class, attr) - if config_instance: - raw_value = getattr(config_instance, attr, "") - if is_minutes and raw_value != "": - try: - value = str( - raw_value.total_seconds() / 60.0 - ) - except AttributeError: - value = str(raw_value) - else: - value = str(raw_value) - else: - value = "" - label = f"{attr.replace('_', ' ').title()}:" - yield Label( - label - if not is_minutes - else label.replace(":", " Minutes:") - ) - yield Input( - id=f"{instrument_name}_{attr}", - type=type_to_textual( - get_field_type(config_class, attr) - ), - validators=[ - Function( - validator, - f"INVALID: value must be {validator.__doc__.lower()}", - ) - for validator in validators - ], - value=value, - ) - yield Label( - "", - id=f"validation-failure-label-{instrument_name}_{attr}", - classes="-hidden validation-failure", - ) + else: + new_wp = Waypoint( + location=Location(latitude=0.0, longitude=0.0), + time=None, + instrument=[], + ) + self.expedition.schedule.waypoints.append(new_wp) + self.refresh_waypoint_widgets() + + except Exception as e: + raise UnexpectedError(unexpected_msg_compose(e)) from None + + @on(Button.Pressed, "#remove_waypoint") + def remove_waypoint(self) -> None: + """Remove the last waypoint from the schedule.""" + try: + if self.expedition.schedule.waypoints: + self.expedition.schedule.waypoints.pop() + self.refresh_waypoint_widgets() + else: + self.notify("No waypoints to remove.", severity="error", timeout=5) except Exception as e: raise UnexpectedError(unexpected_msg_compose(e)) from None @@ -885,31 +724,6 @@ def info_pressed(self) -> None: timeout=20, ) - @on(Input.Changed) - def show_invalid_reasons(self, event: Input.Changed) -> None: - input_id = event.input.id - label_id = f"validation-failure-label-{input_id}" - label = self.query_one(f"#{label_id}", Label) - if not event.validation_result.is_valid: - message = ( - "\n".join(event.validation_result.failure_descriptions) - if isinstance(event.validation_result.failure_descriptions, list) - else str(event.validation_result.failure_descriptions) - ) - label.update(message) - label.remove_class("-hidden") - label.add_class("validation-failure") - else: - label.update("") - label.add_class("-hidden") - label.remove_class("validation-failure") - - def on_mount(self) -> None: - adcp_present = ( - getattr(self.config, "adcp_config", None) if self.config else False - ) - self.show_hide_adcp_type(bool(adcp_present)) - def show_hide_adcp_type(self, show: bool) -> None: container = self.query_one("#adcp_type_container") if show: @@ -919,29 +733,32 @@ def show_hide_adcp_type(self, show: bool) -> None: def _set_adcp_default_values(self): self.query_one("#adcp_config_num_bins").value = str( - self.DEFAULT_ADCP_CONFIG["num_bins"] + DEFAULT_ADCP_CONFIG["num_bins"] ) self.query_one("#adcp_config_period").value = str( - self.DEFAULT_ADCP_CONFIG["period_minutes"] + DEFAULT_ADCP_CONFIG["period_minutes"] ) self.query_one("#adcp_shallow").value = False self.query_one("#adcp_deep").value = True def _set_ts_default_values(self): self.query_one("#ship_underwater_st_config_period").value = str( - self.DEFAULT_TS_CONFIG["period_minutes"] + DEFAULT_TS_CONFIG["period_minutes"] ) @on(Switch.Changed, "#has_adcp") def on_adcp_toggle(self, event: Switch.Changed) -> None: self.show_hide_adcp_type(event.value) - if event.value and not self.config.adcp_config: + if event.value and not self.expedition.instruments_config.adcp_config: # ADCP was turned on and was previously null self._set_adcp_default_values() @on(Switch.Changed, "#has_onboard_ts") def on_ts_toggle(self, event: Switch.Changed) -> None: - if event.value and not self.config.ship_underwater_st_config: + if ( + event.value + and not self.expedition.instruments_config.ship_underwater_st_config + ): # T/S was turned on and was previously null self._set_ts_default_values() @@ -957,68 +774,212 @@ def shallow_changed(self, event: Switch.Changed) -> None: deep = self.query_one("#adcp_deep", Switch) deep.value = False - def save_changes(self) -> bool: - """Save changes to ship_config.yaml.""" + +class WaypointWidget(Static): + def __init__(self, waypoint: Waypoint, index: int): + super().__init__() + self.waypoint = waypoint + self.index = index + + def compose(self) -> ComposeResult: try: - # ship speed - attr = "ship_speed_knots" - field_type = get_field_type(type(self.config), attr) - value = field_type(self.query_one("#speed").value) - ShipConfig.model_validate( - {**self.config.model_dump(), attr: value} - ) # validate using a temporary model (raises if invalid) - self.config.ship_speed_knots = value - - # individual instrument configurations - for instrument_name, info in self.INSTRUMENT_FIELDS.items(): - config_class = info["class"] - attributes = info["attributes"] - kwargs = {} - - # special handling for onboard ADCP and T/S - # will skip to next instrument if toggle is off - if instrument_name == "adcp_config": - has_adcp = self.query_one("#has_adcp", Switch).value - if not has_adcp: - setattr(self.config, instrument_name, None) - continue - if instrument_name == "ship_underwater_st_config": - has_ts = self.query_one("#has_onboard_ts", Switch).value - if not has_ts: - setattr(self.config, instrument_name, None) - continue - - for attr_meta in attributes: - attr = attr_meta["name"] - is_minutes = attr_meta.get("minutes", False) - input_id = f"{instrument_name}_{attr}" - value = self.query_one(f"#{input_id}").value - field_type = get_field_type(config_class, attr) - if is_minutes and field_type is datetime.timedelta: - value = datetime.timedelta(minutes=float(value)) - else: - value = field_type(value) - kwargs[attr] = value - - # ADCP max_depth_meter based on deep/shallow switch - if instrument_name == "adcp_config": - if self.query_one("#adcp_deep", Switch).value: - kwargs["max_depth_meter"] = -1000.0 - else: - kwargs["max_depth_meter"] = -150.0 - - setattr(self.config, instrument_name, config_class(**kwargs)) - - # save - self.config.to_yaml(f"{self.path}/ship_config.yaml") - return True + with Collapsible( + title=f"[b]Waypoint {self.index + 1}[/b]", + collapsed=True, + id=f"wp{self.index + 1}", + ): + if self.index > 0: + yield Button( + "Copy Time & Instruments from Previous", + id=f"wp{self.index}_copy", + variant="warning", + ) + yield Label("Location:") + yield Label(" Latitude:") + yield Input( + id=f"wp{self.index}_lat", + value=str(self.waypoint.location.lat) + if self.waypoint.location.lat + is not None # is not None to handle if lat is 0.0 + else "", + validators=[ + Function( + is_valid_lat, + f"INVALID: value must be {is_valid_lat.__doc__.lower()}", + ) + ], + type="number", + placeholder="°N", + classes="latitude-input", + ) + yield Label( + "", + id=f"validation-failure-label-wp{self.index}_lat", + classes="-hidden validation-failure", + ) + + yield Label(" Longitude:") + yield Input( + id=f"wp{self.index}_lon", + value=str(self.waypoint.location.lon) + if self.waypoint.location.lon + is not None # is not None to handle if lon is 0.0 + else "", + validators=[ + Function( + is_valid_lon, + f"INVALID: value must be {is_valid_lon.__doc__.lower()}", + ) + ], + type="number", + placeholder="°E", + classes="longitude-input", + ) + yield Label( + "", + id=f"validation-failure-label-wp{self.index}_lon", + classes="-hidden validation-failure", + ) + + yield Label("Time:") + with Horizontal(): + yield Label("Year:") + yield Select( + [ + (str(year), year) + # TODO: change from hard coding? ...flexibility for different datasets... + for year in range( + 2022, + datetime.datetime.now().year + 1, + ) + ], + id=f"wp{self.index}_year", + value=int(self.waypoint.time.year) + if self.waypoint.time + else Select.BLANK, + prompt="YYYY", + classes="year-select", + ) + yield Label("Month:") + yield Select( + [(f"{m:02d}", m) for m in range(1, 13)], + id=f"wp{self.index}_month", + value=int(self.waypoint.time.month) + if self.waypoint.time + else Select.BLANK, + prompt="MM", + classes="month-select", + ) + yield Label("Day:") + yield Select( + [(f"{d:02d}", d) for d in range(1, 32)], + id=f"wp{self.index}_day", + value=int(self.waypoint.time.day) + if self.waypoint.time + else Select.BLANK, + prompt="DD", + classes="day-select", + ) + yield Label("Hour:") + yield Select( + [(f"{h:02d}", h) for h in range(24)], + id=f"wp{self.index}_hour", + value=int(self.waypoint.time.hour) + if self.waypoint.time + else Select.BLANK, + prompt="hh", + classes="hour-select", + ) + yield Label("Min:") + yield Select( + [(f"{m:02d}", m) for m in range(0, 60, 5)], + id=f"wp{self.index}_minute", + value=int(self.waypoint.time.minute) + if self.waypoint.time + else Select.BLANK, + prompt="mm", + classes="minute-select", + ) + + yield Label("Instruments:") + for instrument in InstrumentType: + is_selected = instrument in (self.waypoint.instrument or []) + with Horizontal(): + yield Label(instrument.value) + yield Switch( + value=is_selected, id=f"wp{self.index}_{instrument.value}" + ) + + if instrument.value == "DRIFTER": + yield Label("Count") + yield Input( + id=f"wp{self.index}_drifter_count", + value=str( + self.get_drifter_count() if is_selected else "" + ), + type="integer", + placeholder="# of drifters", + validators=Integer( + minimum=1, + failure_description="INVALID: value must be > 0", + ), + classes="drifter-count-input", + ) + yield Label( + "", + id=f"validation-failure-label-wp{self.index}_drifter_count", + classes="-hidden validation-failure", + ) except Exception as e: - log_exception_to_file( - e, self.path, context_message="Error saving ship config:" - ) + raise UnexpectedError(unexpected_msg_compose(e)) from None + + def get_drifter_count(self) -> int: + return sum( + 1 for inst in self.waypoint.instrument if inst == InstrumentType.DRIFTER + ) + + def copy_from_previous(self) -> None: + """Copy inputs from previous waypoint widget (time and instruments only, not lat/lon).""" + try: + if self.index > 0: + schedule_editor = self.parent + if schedule_editor: + time_components = ["year", "month", "day", "hour", "minute"] + for comp in time_components: + prev = schedule_editor.query_one(f"#wp{self.index - 1}_{comp}") + curr = self.query_one(f"#wp{self.index}_{comp}") + if prev and curr: + curr.value = prev.value + + for instrument in InstrumentType: + prev_switch = schedule_editor.query_one( + f"#wp{self.index - 1}_{instrument.value}" + ) + curr_switch = self.query_one( + f"#wp{self.index}_{instrument.value}" + ) + if prev_switch and curr_switch: + curr_switch.value = prev_switch.value + except Exception as e: + raise UnexpectedError(unexpected_msg_compose(e)) from None + + @on(Button.Pressed, "Button") + def button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == f"wp{self.index}_copy": + self.copy_from_previous() - raise UnexpectedError(UNEXPECTED_MSG_ONSAVE) from None + @on(Switch.Changed) + def on_switch_changed(self, event: Switch.Changed) -> None: + if event.switch.id == f"wp{self.index}_DRIFTER": + drifter_count_input = self.query_one( + f"#wp{self.index}_drifter_count", Input + ) + if not event.value: + drifter_count_input.value = "" + else: + if not drifter_count_input.value: + drifter_count_input.value = "1" class PlanScreen(Screen): @@ -1029,8 +990,7 @@ def __init__(self, path: str): def compose(self) -> ComposeResult: try: with VerticalScroll(): - yield ConfigEditor(self.path) - yield ScheduleEditor(self.path) + yield ExpeditionEditor(self.path) with Horizontal(): yield Button("Save Changes", id="save_button", variant="success") yield Button("Exit", id="exit_button", variant="error") @@ -1039,20 +999,20 @@ def compose(self) -> ComposeResult: def sync_ui_waypoints(self): """Update the waypoints models with current UI values (spacetime only) from the live UI inputs.""" - schedule_editor = self.query_one(ScheduleEditor) + expedition_editor = self.query_one(ExpeditionEditor) errors = [] - for i, wp in enumerate(schedule_editor.schedule.waypoints): + for i, wp in enumerate(expedition_editor.expedition.schedule.waypoints): try: wp.location = Location( - latitude=float(schedule_editor.query_one(f"#wp{i}_lat").value), - longitude=float(schedule_editor.query_one(f"#wp{i}_lon").value), + latitude=float(expedition_editor.query_one(f"#wp{i}_lat").value), + longitude=float(expedition_editor.query_one(f"#wp{i}_lon").value), ) wp.time = datetime.datetime( - int(schedule_editor.query_one(f"#wp{i}_year").value), - int(schedule_editor.query_one(f"#wp{i}_month").value), - int(schedule_editor.query_one(f"#wp{i}_day").value), - int(schedule_editor.query_one(f"#wp{i}_hour").value), - int(schedule_editor.query_one(f"#wp{i}_minute").value), + int(expedition_editor.query_one(f"#wp{i}_year").value), + int(expedition_editor.query_one(f"#wp{i}_month").value), + int(expedition_editor.query_one(f"#wp{i}_day").value), + int(expedition_editor.query_one(f"#wp{i}_hour").value), + int(expedition_editor.query_one(f"#wp{i}_minute").value), 0, ) except Exception as e: @@ -1075,26 +1035,24 @@ def exit_pressed(self) -> None: @on(Button.Pressed, "#save_button") def save_pressed(self) -> None: """Save button press.""" - config_editor = self.query_one(ConfigEditor) - schedule_editor = self.query_one(ScheduleEditor) + expedition_editor = self.query_one(ExpeditionEditor) try: - ship_speed_value = self.get_ship_speed(config_editor) + ship_speed_value = self.get_ship_speed(expedition_editor) self.sync_ui_waypoints() # call to ensure waypoint inputs are synced # verify schedule - schedule_editor.schedule.verify( + expedition_editor.expedition.schedule.verify( ship_speed_value, input_data=None, check_space_time_region=True, ignore_missing_fieldsets=True, ) - config_saved = config_editor.save_changes() - schedule_saved = schedule_editor.save_changes() + expedition_saved = expedition_editor.save_changes() - if config_saved and schedule_saved: + if expedition_saved: self.notify( "Changes saved successfully", severity="information", @@ -1109,9 +1067,9 @@ def save_pressed(self) -> None: ) return False - def get_ship_speed(self, config_editor): + def get_ship_speed(self, expedition_editor): try: - ship_speed = float(config_editor.query_one("#speed").value) + ship_speed = float(expedition_editor.query_one("#speed").value) assert ship_speed > 0 except Exception as e: log_exception_to_file( @@ -1130,12 +1088,6 @@ class PlanApp(App): align: center middle; } - ConfigEditor { - padding: 1; - margin-bottom: 1; - height: auto; - } - VerticalScroll { width: 100%; height: 100%; @@ -1210,7 +1162,12 @@ class PlanApp(App): margin: 0 1; } - #title { + #title_ship_instruments_config { + text-style: bold; + padding: 1; + } + + #title_schedule { text-style: bold; padding: 1; } diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index 72d37866..3e83be3b 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -7,8 +7,7 @@ from virtualship.cli._plan import _plan from virtualship.expedition.do_expedition import do_expedition from virtualship.utils import ( - SCHEDULE, - SHIP_CONFIG, + EXPEDITION, mfp_to_yaml, ) @@ -28,47 +27,39 @@ ) def init(path, from_mfp): """ - Initialize a directory for a new expedition, with an example schedule and ship config files. + Initialize a directory for a new expedition, with an expedition.yaml file. - If --mfp-file is provided, it will generate the schedule from the MPF file instead. + If --mfp-file is provided, it will generate the expedition.yaml from the MPF file instead. """ path = Path(path) path.mkdir(exist_ok=True) - config = path / SHIP_CONFIG - schedule = path / SCHEDULE + expedition = path / EXPEDITION - if config.exists(): + if expedition.exists(): raise FileExistsError( - f"File '{config}' already exist. Please remove it or choose another directory." + f"File '{expedition}' already exist. Please remove it or choose another directory." ) - if schedule.exists(): - raise FileExistsError( - f"File '{schedule}' already exist. Please remove it or choose another directory." - ) - - config.write_text(utils.get_example_config()) if from_mfp: mfp_file = Path(from_mfp) - # Generate schedule.yaml from the MPF file + # Generate expedition.yaml from the MPF file click.echo(f"Generating schedule from {mfp_file}...") - mfp_to_yaml(mfp_file, schedule) + mfp_to_yaml(mfp_file, expedition) click.echo( "\n⚠️ The generated schedule does not contain TIME values or INSTRUMENT selections. ⚠️" "\n\nNow please either use the `\033[4mvirtualship plan\033[0m` app to complete the schedule configuration, " - "\nOR edit 'schedule.yaml' and manually add the necessary time values and instrument selections." - "\n\nIf editing 'schedule.yaml' manually:" + "\nOR edit 'expedition.yaml' and manually add the necessary time values and instrument selections under the 'schedule' heading." + "\n\nIf editing 'expedition.yaml' manually:" "\n\n🕒 Expected time format: 'YYYY-MM-DD HH:MM:SS' (e.g., '2023-10-20 01:00:00')." "\n\n🌡️ Expected instrument(s) format: one line per instrument e.g." f"\n\n{' ' * 15}waypoints:\n{' ' * 15}- instrument:\n{' ' * 19}- CTD\n{' ' * 19}- ARGO_FLOAT\n" ) else: - # Create a default example schedule - # schedule_body = utils.get_example_schedule() - schedule.write_text(utils.get_example_schedule()) + # Create a default example expedition YAML + expedition.write_text(utils.get_example_expedition()) - click.echo(f"Created '{config.name}' and '{schedule.name}' at {path}.") + click.echo(f"Created '{expedition.name}' at {path}.") @click.command() diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index 56ee79fa..5c46d2eb 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -7,11 +7,10 @@ import pyproj from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash -from virtualship.models import Schedule, ShipConfig +from virtualship.models import Expedition, Schedule from virtualship.utils import ( CHECKPOINT, - _get_schedule, - _get_ship_config, + _get_expedition, ) from .checkpoint import Checkpoint @@ -38,11 +37,10 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> if isinstance(expedition_dir, str): expedition_dir = Path(expedition_dir) - ship_config = _get_ship_config(expedition_dir) - schedule = _get_schedule(expedition_dir) + expedition = _get_expedition(expedition_dir) - # Verify ship_config file is consistent with schedule - ship_config.verify(schedule) + # Verify instruments_config file is consistent with schedule + expedition.instruments_config.verify(expedition.schedule) # load last checkpoint checkpoint = _load_checkpoint(expedition_dir) @@ -50,24 +48,26 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> checkpoint = Checkpoint(past_schedule=Schedule(waypoints=[])) # verify that schedule and checkpoint match - checkpoint.verify(schedule) + checkpoint.verify(expedition.schedule) # load fieldsets loaded_input_data = _load_input_data( expedition_dir=expedition_dir, - schedule=schedule, - ship_config=ship_config, + expedition=expedition, input_data=input_data, ) print("\n---- WAYPOINT VERIFICATION ----") # verify schedule is valid - schedule.verify(ship_config.ship_speed_knots, loaded_input_data) + expedition.schedule.verify( + expedition.ship_config.ship_speed_knots, loaded_input_data + ) # simulate the schedule schedule_results = simulate_schedule( - projection=projection, ship_config=ship_config, schedule=schedule + projection=projection, + expedition=expedition, ) if isinstance(schedule_results, ScheduleProblem): print( @@ -76,7 +76,9 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> _save_checkpoint( Checkpoint( past_schedule=Schedule( - waypoints=schedule.waypoints[: schedule_results.failed_waypoint_i] + waypoints=expedition.schedule.waypoints[ + : schedule_results.failed_waypoint_i + ] ) ), expedition_dir, @@ -91,10 +93,10 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> print("\n----- EXPEDITION SUMMARY ------") # calculate expedition cost in US$ - assert schedule.waypoints[0].time is not None, ( + assert expedition.schedule.waypoints[0].time is not None, ( "First waypoint has no time. This should not be possible as it should have been verified before." ) - time_past = schedule_results.time - schedule.waypoints[0].time + time_past = schedule_results.time - expedition.schedule.waypoints[0].time cost = expedition_cost(schedule_results, time_past) with open(expedition_dir.joinpath("results", "cost.txt"), "w") as file: file.writelines(f"cost: {cost} US$") @@ -106,7 +108,7 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> print("\nSimulating measurements. This may take a while...\n") simulate_measurements( expedition_dir, - ship_config, + expedition.instruments_config, loaded_input_data, schedule_results.measurements_to_simulate, ) @@ -122,26 +124,21 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> def _load_input_data( expedition_dir: Path, - schedule: Schedule, - ship_config: ShipConfig, + expedition: Expedition, input_data: Path | None, ) -> InputData: """ Load the input data. :param expedition_dir: Directory of the expedition. - :type expedition_dir: Path - :param schedule: Schedule object. - :type schedule: Schedule - :param ship_config: Ship configuration. - :type ship_config: ShipConfig + :param expedition: Expedition object. :param input_data: Folder containing input data. - :type input_data: Path | None :return: InputData object. - :rtype: InputData """ if input_data is None: - space_time_region_hash = get_space_time_region_hash(schedule.space_time_region) + space_time_region_hash = get_space_time_region_hash( + expedition.schedule.space_time_region + ) input_data = get_existing_download(expedition_dir, space_time_region_hash) assert input_data is not None, ( @@ -150,13 +147,14 @@ def _load_input_data( return InputData.load( directory=input_data, - load_adcp=ship_config.adcp_config is not None, - load_argo_float=ship_config.argo_float_config is not None, - load_ctd=ship_config.ctd_config is not None, - load_ctd_bgc=ship_config.ctd_bgc_config is not None, - load_drifter=ship_config.drifter_config is not None, - load_xbt=ship_config.xbt_config is not None, - load_ship_underwater_st=ship_config.ship_underwater_st_config is not None, + load_adcp=expedition.instruments_config.adcp_config is not None, + load_argo_float=expedition.instruments_config.argo_float_config is not None, + load_ctd=expedition.instruments_config.ctd_config is not None, + load_ctd_bgc=expedition.instruments_config.ctd_bgc_config is not None, + load_drifter=expedition.instruments_config.drifter_config is not None, + load_xbt=expedition.instruments_config.xbt_config is not None, + load_ship_underwater_st=expedition.instruments_config.ship_underwater_st_config + is not None, ) diff --git a/src/virtualship/expedition/simulate_measurements.py b/src/virtualship/expedition/simulate_measurements.py index 20ba2cdb..6cb2e488 100644 --- a/src/virtualship/expedition/simulate_measurements.py +++ b/src/virtualship/expedition/simulate_measurements.py @@ -16,7 +16,7 @@ from virtualship.instruments.drifter import simulate_drifters from virtualship.instruments.ship_underwater_st import simulate_ship_underwater_st from virtualship.instruments.xbt import simulate_xbt -from virtualship.models import ShipConfig +from virtualship.models import InstrumentsConfig from virtualship.utils import ship_spinner from .simulate_schedule import MeasurementsToSimulate @@ -31,7 +31,7 @@ def simulate_measurements( expedition_dir: str | Path, - ship_config: ShipConfig, + instruments_config: InstrumentsConfig, input_data: InputData, measurements: MeasurementsToSimulate, ) -> None: @@ -41,7 +41,6 @@ def simulate_measurements( Saves everything in expedition_dir/results. :param expedition_dir: Base directory of the expedition. - :param ship_config: Ship configuration. :param input_data: Input data for simulation. :param measurements: The measurements to simulate. :raises RuntimeError: In case fieldsets of configuration is not provided. Make sure to check this before calling this function. @@ -50,7 +49,7 @@ def simulate_measurements( expedition_dir = Path(expedition_dir) if len(measurements.ship_underwater_sts) > 0: - if ship_config.ship_underwater_st_config is None: + if instruments_config.ship_underwater_st_config is None: raise RuntimeError("No configuration for ship underwater ST provided.") if input_data.ship_underwater_st_fieldset is None: raise RuntimeError("No fieldset for ship underwater ST provided.") @@ -68,7 +67,7 @@ def simulate_measurements( spinner.ok("✅") if len(measurements.adcps) > 0: - if ship_config.adcp_config is None: + if instruments_config.adcp_config is None: raise RuntimeError("No configuration for ADCP provided.") if input_data.adcp_fieldset is None: raise RuntimeError("No fieldset for ADCP provided.") @@ -78,15 +77,15 @@ def simulate_measurements( simulate_adcp( fieldset=input_data.adcp_fieldset, out_path=expedition_dir.joinpath("results", "adcp.zarr"), - max_depth=ship_config.adcp_config.max_depth_meter, + max_depth=instruments_config.adcp_config.max_depth_meter, min_depth=-5, - num_bins=ship_config.adcp_config.num_bins, + num_bins=instruments_config.adcp_config.num_bins, sample_points=measurements.adcps, ) spinner.ok("✅") if len(measurements.ctds) > 0: - if ship_config.ctd_config is None: + if instruments_config.ctd_config is None: raise RuntimeError("No configuration for CTD provided.") if input_data.ctd_fieldset is None: raise RuntimeError("No fieldset for CTD provided.") @@ -102,7 +101,7 @@ def simulate_measurements( spinner.ok("✅") if len(measurements.ctd_bgcs) > 0: - if ship_config.ctd_bgc_config is None: + if instruments_config.ctd_bgc_config is None: raise RuntimeError("No configuration for CTD_BGC provided.") if input_data.ctd_bgc_fieldset is None: raise RuntimeError("No fieldset for CTD_BGC provided.") @@ -118,7 +117,7 @@ def simulate_measurements( spinner.ok("✅") if len(measurements.xbts) > 0: - if ship_config.xbt_config is None: + if instruments_config.xbt_config is None: raise RuntimeError("No configuration for XBTs provided.") if input_data.xbt_fieldset is None: raise RuntimeError("No fieldset for XBTs provided.") @@ -135,7 +134,7 @@ def simulate_measurements( if len(measurements.drifters) > 0: print("Simulating drifters... ") - if ship_config.drifter_config is None: + if instruments_config.drifter_config is None: raise RuntimeError("No configuration for drifters provided.") if input_data.drifter_fieldset is None: raise RuntimeError("No fieldset for drifters provided.") @@ -150,7 +149,7 @@ def simulate_measurements( if len(measurements.argo_floats) > 0: print("Simulating argo floats... ") - if ship_config.argo_float_config is None: + if instruments_config.argo_float_config is None: raise RuntimeError("No configuration for argo floats provided.") if input_data.argo_float_fieldset is None: raise RuntimeError("No fieldset for argo floats provided.") diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 95fa2f5f..3b78c5c7 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -13,10 +13,9 @@ from virtualship.instruments.drifter import Drifter from virtualship.instruments.xbt import XBT from virtualship.models import ( + Expedition, InstrumentType, Location, - Schedule, - ShipConfig, Spacetime, Waypoint, ) @@ -52,23 +51,21 @@ class MeasurementsToSimulate: def simulate_schedule( - projection: pyproj.Geod, ship_config: ShipConfig, schedule: Schedule + projection: pyproj.Geod, expedition: Expedition ) -> ScheduleOk | ScheduleProblem: """ Simulate a schedule. :param projection: The projection to use for sailing. - :param ship_config: Ship configuration. - :param schedule: The schedule to simulate. + :param expedition: Expedition object containing the schedule to simulate. :returns: Either the results of a successfully simulated schedule, or information on where the schedule became infeasible. """ - return _ScheduleSimulator(projection, ship_config, schedule).simulate() + return _ScheduleSimulator(projection, expedition).simulate() class _ScheduleSimulator: _projection: pyproj.Geod - _ship_config: ShipConfig - _schedule: Schedule + _expedition: Expedition _time: datetime """Current time.""" @@ -82,18 +79,15 @@ class _ScheduleSimulator: _next_ship_underwater_st_time: datetime """Next moment ship underwater ST measurement will be done.""" - def __init__( - self, projection: pyproj.Geod, ship_config: ShipConfig, schedule: Schedule - ) -> None: + def __init__(self, projection: pyproj.Geod, expedition: Expedition) -> None: self._projection = projection - self._ship_config = ship_config - self._schedule = schedule + self._expedition = expedition - assert self._schedule.waypoints[0].time is not None, ( + assert self._expedition.schedule.waypoints[0].time is not None, ( "First waypoint must have a time. This should have been verified before calling this function." ) - self._time = schedule.waypoints[0].time - self._location = schedule.waypoints[0].location + self._time = expedition.schedule.waypoints[0].time + self._location = expedition.schedule.waypoints[0].location self._measurements_to_simulate = MeasurementsToSimulate() @@ -101,7 +95,7 @@ def __init__( self._next_ship_underwater_st_time = self._time def simulate(self) -> ScheduleOk | ScheduleProblem: - for wp_i, waypoint in enumerate(self._schedule.waypoints): + for wp_i, waypoint in enumerate(self._expedition.schedule.waypoints): # sail towards waypoint self._progress_time_traveling_towards(waypoint.location) @@ -131,7 +125,9 @@ def _progress_time_traveling_towards(self, location: Location) -> None: lons2=location.lon, lats2=location.lat, ) - ship_speed_meter_per_second = self._ship_config.ship_speed_knots * 1852 / 3600 + ship_speed_meter_per_second = ( + self._expedition.ship_config.ship_speed_knots * 1852 / 3600 + ) azimuth1 = geodinv[0] distance_to_next_waypoint = geodinv[2] time_to_reach = timedelta( @@ -140,7 +136,7 @@ def _progress_time_traveling_towards(self, location: Location) -> None: end_time = self._time + time_to_reach # note all ADCP measurements - if self._ship_config.adcp_config is not None: + if self._expedition.instruments_config.adcp_config is not None: location = self._location time = self._time while self._next_adcp_time <= end_time: @@ -162,11 +158,12 @@ def _progress_time_traveling_towards(self, location: Location) -> None: ) self._next_adcp_time = ( - self._next_adcp_time + self._ship_config.adcp_config.period + self._next_adcp_time + + self._expedition.instruments_config.adcp_config.period ) # note all ship underwater ST measurements - if self._ship_config.ship_underwater_st_config is not None: + if self._expedition.instruments_config.ship_underwater_st_config is not None: location = self._location time = self._time while self._next_ship_underwater_st_time <= end_time: @@ -189,7 +186,7 @@ def _progress_time_traveling_towards(self, location: Location) -> None: self._next_ship_underwater_st_time = ( self._next_ship_underwater_st_time - + self._ship_config.ship_underwater_st_config.period + + self._expedition.instruments_config.ship_underwater_st_config.period ) self._time = end_time @@ -199,24 +196,25 @@ def _progress_time_stationary(self, time_passed: timedelta) -> None: end_time = self._time + time_passed # note all ADCP measurements - if self._ship_config.adcp_config is not None: + if self._expedition.instruments_config.adcp_config is not None: while self._next_adcp_time <= end_time: self._measurements_to_simulate.adcps.append( Spacetime(self._location, self._next_adcp_time) ) self._next_adcp_time = ( - self._next_adcp_time + self._ship_config.adcp_config.period + self._next_adcp_time + + self._expedition.instruments_config.adcp_config.period ) # note all ship underwater ST measurements - if self._ship_config.ship_underwater_st_config is not None: + if self._expedition.instruments_config.ship_underwater_st_config is not None: while self._next_ship_underwater_st_time <= end_time: self._measurements_to_simulate.ship_underwater_sts.append( Spacetime(self._location, self._next_ship_underwater_st_time) ) self._next_ship_underwater_st_time = ( self._next_ship_underwater_st_time - + self._ship_config.ship_underwater_st_config.period + + self._expedition.instruments_config.ship_underwater_st_config.period ) self._time = end_time @@ -241,48 +239,52 @@ def _make_measurements(self, waypoint: Waypoint) -> timedelta: self._measurements_to_simulate.argo_floats.append( ArgoFloat( spacetime=Spacetime(self._location, self._time), - min_depth=self._ship_config.argo_float_config.min_depth_meter, - max_depth=self._ship_config.argo_float_config.max_depth_meter, - drift_depth=self._ship_config.argo_float_config.drift_depth_meter, - vertical_speed=self._ship_config.argo_float_config.vertical_speed_meter_per_second, - cycle_days=self._ship_config.argo_float_config.cycle_days, - drift_days=self._ship_config.argo_float_config.drift_days, + min_depth=self._expedition.instruments_config.argo_float_config.min_depth_meter, + max_depth=self._expedition.instruments_config.argo_float_config.max_depth_meter, + drift_depth=self._expedition.instruments_config.argo_float_config.drift_depth_meter, + vertical_speed=self._expedition.instruments_config.argo_float_config.vertical_speed_meter_per_second, + cycle_days=self._expedition.instruments_config.argo_float_config.cycle_days, + drift_days=self._expedition.instruments_config.argo_float_config.drift_days, ) ) elif instrument is InstrumentType.CTD: self._measurements_to_simulate.ctds.append( CTD( spacetime=Spacetime(self._location, self._time), - min_depth=self._ship_config.ctd_config.min_depth_meter, - max_depth=self._ship_config.ctd_config.max_depth_meter, + min_depth=self._expedition.instruments_config.ctd_config.min_depth_meter, + max_depth=self._expedition.instruments_config.ctd_config.max_depth_meter, ) ) - time_costs.append(self._ship_config.ctd_config.stationkeeping_time) + time_costs.append( + self._expedition.instruments_config.ctd_config.stationkeeping_time + ) elif instrument is InstrumentType.CTD_BGC: self._measurements_to_simulate.ctd_bgcs.append( CTD_BGC( spacetime=Spacetime(self._location, self._time), - min_depth=self._ship_config.ctd_bgc_config.min_depth_meter, - max_depth=self._ship_config.ctd_bgc_config.max_depth_meter, + min_depth=self._expedition.instruments_config.ctd_bgc_config.min_depth_meter, + max_depth=self._expedition.instruments_config.ctd_bgc_config.max_depth_meter, ) ) - time_costs.append(self._ship_config.ctd_bgc_config.stationkeeping_time) + time_costs.append( + self._expedition.instruments_config.ctd_bgc_config.stationkeeping_time + ) elif instrument is InstrumentType.DRIFTER: self._measurements_to_simulate.drifters.append( Drifter( spacetime=Spacetime(self._location, self._time), - depth=self._ship_config.drifter_config.depth_meter, - lifetime=self._ship_config.drifter_config.lifetime, + depth=self._expedition.instruments_config.drifter_config.depth_meter, + lifetime=self._expedition.instruments_config.drifter_config.lifetime, ) ) elif instrument is InstrumentType.XBT: self._measurements_to_simulate.xbts.append( XBT( spacetime=Spacetime(self._location, self._time), - min_depth=self._ship_config.xbt_config.min_depth_meter, - max_depth=self._ship_config.xbt_config.max_depth_meter, - fall_speed=self._ship_config.xbt_config.fall_speed_meter_per_second, - deceleration_coefficient=self._ship_config.xbt_config.deceleration_coefficient, + min_depth=self._expedition.instruments_config.xbt_config.min_depth_meter, + max_depth=self._expedition.instruments_config.xbt_config.max_depth_meter, + fall_speed=self._expedition.instruments_config.xbt_config.fall_speed_meter_per_second, + deceleration_coefficient=self._expedition.instruments_config.xbt_config.deceleration_coefficient, ) ) else: diff --git a/src/virtualship/models/__init__.py b/src/virtualship/models/__init__.py index 48106056..a2f1546c 100644 --- a/src/virtualship/models/__init__.py +++ b/src/virtualship/models/__init__.py @@ -1,18 +1,21 @@ """Pydantic models and data classes used to configure virtualship (i.e., in the configuration files or settings).""" -from .location import Location -from .schedule import Schedule, Waypoint -from .ship_config import ( +from .expedition import ( ADCPConfig, ArgoFloatConfig, CTD_BGCConfig, CTDConfig, DrifterConfig, + Expedition, + InstrumentsConfig, InstrumentType, + Schedule, ShipConfig, ShipUnderwaterSTConfig, + Waypoint, XBTConfig, ) +from .location import Location from .space_time_region import ( SpaceTimeRegion, SpatialRange, @@ -25,6 +28,7 @@ __all__ = [ # noqa: RUF022 "Location", "Schedule", + "ShipConfig", "Waypoint", "InstrumentType", "ArgoFloatConfig", @@ -34,9 +38,10 @@ "ShipUnderwaterSTConfig", "DrifterConfig", "XBTConfig", - "ShipConfig", "SpatialRange", "TimeRange", "SpaceTimeRegion", "Spacetime", + "Expedition", + "InstrumentsConfig", ] diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py new file mode 100644 index 00000000..2e073b84 --- /dev/null +++ b/src/virtualship/models/expedition.py @@ -0,0 +1,456 @@ +from __future__ import annotations + +import itertools +from datetime import datetime, timedelta +from enum import Enum +from typing import TYPE_CHECKING + +import pydantic +import pyproj +import yaml + +from virtualship.errors import ConfigError, ScheduleError +from virtualship.utils import _validate_numeric_mins_to_timedelta + +from .location import Location +from .space_time_region import SpaceTimeRegion + +if TYPE_CHECKING: + from parcels import FieldSet + + from virtualship.expedition.input_data import InputData + + +projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") + + +class Expedition(pydantic.BaseModel): + """Expedition class, including schedule and ship config.""" + + schedule: Schedule + instruments_config: InstrumentsConfig + ship_config: ShipConfig + + model_config = pydantic.ConfigDict(extra="forbid") + + def to_yaml(self, file_path: str) -> None: + """Write exepedition object to yaml file.""" + with open(file_path, "w") as file: + yaml.dump(self.model_dump(by_alias=True), file) + + @classmethod + def from_yaml(cls, file_path: str) -> Expedition: + """Load config from yaml file.""" + with open(file_path) as file: + data = yaml.safe_load(file) + return Expedition(**data) + + +class ShipConfig(pydantic.BaseModel): + """Configuration of the ship.""" + + ship_speed_knots: float = pydantic.Field(gt=0.0) + + # TODO: room here for adding more ship config options in future PRs (e.g. max_days_at_sea)... + + model_config = pydantic.ConfigDict(extra="forbid") + + +class Schedule(pydantic.BaseModel): + """Schedule of the virtual ship.""" + + waypoints: list[Waypoint] + space_time_region: SpaceTimeRegion | None = None + + model_config = pydantic.ConfigDict(extra="forbid") + + def get_instruments(self) -> set[InstrumentType]: + """Return a set of unique InstrumentType enums used in the schedule.""" + instruments_in_schedule = [] + for waypoint in self.waypoints: + if waypoint.instrument: + for instrument in waypoint.instrument: + if instrument: + instruments_in_schedule.append(instrument) + return set(instruments_in_schedule) + + def verify( + self, + ship_speed: float, + input_data: InputData | None, + *, + check_space_time_region: bool = False, + ignore_missing_fieldsets: bool = False, + ) -> None: + """ + Verify the feasibility and correctness of the schedule's waypoints. + + This method checks various conditions to ensure the schedule is valid: + 1. At least one waypoint is provided. + 2. The first waypoint has a specified time. + 3. Waypoint times are in ascending order. + 4. All waypoints are in water (not on land). + 5. The ship can arrive on time at each waypoint given its speed. + + :param ship_speed: The ship's speed in knots. + :param input_data: An InputData object containing fieldsets used to check if waypoints are on water. + :param check_space_time_region: whether to check for missing space_time_region. + :param ignore_missing_fieldsets: whether to ignore warning for missing field sets. + :raises PlanningError: If any of the verification checks fail, indicating infeasible or incorrect waypoints. + :raises NotImplementedError: If an instrument in the schedule is not implemented. + :return: None. The method doesn't return a value but raises exceptions if verification fails. + """ + print("\nVerifying route... ") + + if check_space_time_region and self.space_time_region is None: + raise ScheduleError( + "space_time_region not found in schedule, please define it to fetch the data." + ) + + if len(self.waypoints) == 0: + raise ScheduleError("At least one waypoint must be provided.") + + # check first waypoint has a time + if self.waypoints[0].time is None: + raise ScheduleError("First waypoint must have a specified time.") + + # check waypoint times are in ascending order + timed_waypoints = [wp for wp in self.waypoints if wp.time is not None] + checks = [ + next.time >= cur.time for cur, next in itertools.pairwise(timed_waypoints) + ] + if not all(checks): + invalid_i = [i for i, c in enumerate(checks) if c] + raise ScheduleError( + f"Waypoint(s) {', '.join(f'#{i + 1}' for i in invalid_i)}: each waypoint should be timed after all previous waypoints", + ) + + # check if all waypoints are in water + # this is done by picking an arbitrary provided fieldset and checking if UV is not zero + + # get all available fieldsets + available_fieldsets = [] + if input_data is not None: + fieldsets = [ + input_data.adcp_fieldset, + input_data.argo_float_fieldset, + input_data.ctd_fieldset, + input_data.drifter_fieldset, + input_data.ship_underwater_st_fieldset, + ] + for fs in fieldsets: + if fs is not None: + available_fieldsets.append(fs) + + # check if there are any fieldsets, else it's an error + if len(available_fieldsets) == 0: + if not ignore_missing_fieldsets: + print( + "Cannot verify because no fieldsets have been loaded. This is probably " + "because you are not using any instruments in your schedule. This is not a problem, " + "but carefully check your waypoint locations manually." + ) + + else: + # pick any + fieldset = available_fieldsets[0] + # get waypoints with 0 UV + land_waypoints = [ + (wp_i, wp) + for wp_i, wp in enumerate(self.waypoints) + if _is_on_land_zero_uv(fieldset, wp) + ] + # raise an error if there are any + if len(land_waypoints) > 0: + raise ScheduleError( + f"The following waypoints are on land: {['#' + str(wp_i) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" + ) + + # check that ship will arrive on time at each waypoint (in case no unexpected event happen) + time = self.waypoints[0].time + for wp_i, (wp, wp_next) in enumerate( + zip(self.waypoints, self.waypoints[1:], strict=False) + ): + if wp.instrument is InstrumentType.CTD: + time += timedelta(minutes=20) + + geodinv: tuple[float, float, float] = projection.inv( + wp.location.lon, + wp.location.lat, + wp_next.location.lon, + wp_next.location.lat, + ) + distance = geodinv[2] + + time_to_reach = timedelta(seconds=distance / ship_speed * 3600 / 1852) + arrival_time = time + time_to_reach + + if wp_next.time is None: + time = arrival_time + elif arrival_time > wp_next.time: + raise ScheduleError( + f"Waypoint planning is not valid: would arrive too late at waypoint number {wp_i + 2}. " + f"location: {wp_next.location} time: {wp_next.time} instrument: {wp_next.instrument}" + ) + else: + time = wp_next.time + + print("... All good to go!") + + +class Waypoint(pydantic.BaseModel): + """A Waypoint to sail to with an optional time and an optional instrument.""" + + location: Location + time: datetime | None = None + instrument: InstrumentType | list[InstrumentType] | None = None + + @pydantic.field_serializer("instrument") + def serialize_instrument(self, instrument): + """Ensure InstrumentType is serialized as a string (or list of strings).""" + if isinstance(instrument, list): + return [inst.value for inst in instrument] + return instrument.value if instrument else None + + +class InstrumentType(Enum): + """Types of the instruments.""" + + CTD = "CTD" + CTD_BGC = "CTD_BGC" + DRIFTER = "DRIFTER" + ARGO_FLOAT = "ARGO_FLOAT" + XBT = "XBT" + + +class ArgoFloatConfig(pydantic.BaseModel): + """Configuration for argos floats.""" + + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + drift_depth_meter: float = pydantic.Field(le=0.0) + vertical_speed_meter_per_second: float = pydantic.Field(lt=0.0) + cycle_days: float = pydantic.Field(gt=0.0) + drift_days: float = pydantic.Field(gt=0.0) + + +class ADCPConfig(pydantic.BaseModel): + """Configuration for ADCP instrument.""" + + max_depth_meter: float = pydantic.Field(le=0.0) + num_bins: int = pydantic.Field(gt=0.0) + period: timedelta = pydantic.Field( + serialization_alias="period_minutes", + validation_alias="period_minutes", + gt=timedelta(), + ) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("period") + def _serialize_period(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("period", mode="before") + def _validate_period(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class CTDConfig(pydantic.BaseModel): + """Configuration for CTD instrument.""" + + stationkeeping_time: timedelta = pydantic.Field( + serialization_alias="stationkeeping_time_minutes", + validation_alias="stationkeeping_time_minutes", + gt=timedelta(), + ) + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("stationkeeping_time") + def _serialize_stationkeeping_time(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("stationkeeping_time", mode="before") + def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class CTD_BGCConfig(pydantic.BaseModel): + """Configuration for CTD_BGC instrument.""" + + stationkeeping_time: timedelta = pydantic.Field( + serialization_alias="stationkeeping_time_minutes", + validation_alias="stationkeeping_time_minutes", + gt=timedelta(), + ) + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("stationkeeping_time") + def _serialize_stationkeeping_time(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("stationkeeping_time", mode="before") + def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class ShipUnderwaterSTConfig(pydantic.BaseModel): + """Configuration for underwater ST.""" + + period: timedelta = pydantic.Field( + serialization_alias="period_minutes", + validation_alias="period_minutes", + gt=timedelta(), + ) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("period") + def _serialize_period(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("period", mode="before") + def _validate_period(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class DrifterConfig(pydantic.BaseModel): + """Configuration for drifters.""" + + depth_meter: float = pydantic.Field(le=0.0) + lifetime: timedelta = pydantic.Field( + serialization_alias="lifetime_minutes", + validation_alias="lifetime_minutes", + gt=timedelta(), + ) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("lifetime") + def _serialize_lifetime(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("lifetime", mode="before") + def _validate_lifetime(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class XBTConfig(pydantic.BaseModel): + """Configuration for xbt instrument.""" + + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + fall_speed_meter_per_second: float = pydantic.Field(gt=0.0) + deceleration_coefficient: float = pydantic.Field(gt=0.0) + + +class InstrumentsConfig(pydantic.BaseModel): + """Configuration of instruments.""" + + argo_float_config: ArgoFloatConfig | None = None + """ + Argo float configuration. + + If None, no argo floats can be deployed. + """ + + adcp_config: ADCPConfig | None = None + """ + ADCP configuration. + + If None, no ADCP measurements will be performed. + """ + + ctd_config: CTDConfig | None = None + """ + CTD configuration. + + If None, no CTDs can be cast. + """ + + ctd_bgc_config: CTD_BGCConfig | None = None + """ + CTD_BGC configuration. + + If None, no BGC CTDs can be cast. + """ + + ship_underwater_st_config: ShipUnderwaterSTConfig | None = None + """ + Ship underwater salinity temperature measurementconfiguration. + + If None, no ST measurements will be performed. + """ + + drifter_config: DrifterConfig | None = None + """ + Drifter configuration. + + If None, no drifters can be deployed. + """ + + xbt_config: XBTConfig | None = None + """ + XBT configuration. + + If None, no XBTs can be cast. + """ + + model_config = pydantic.ConfigDict(extra="forbid") + + def verify(self, schedule: Schedule) -> None: + """ + Verify instrument configurations against the schedule. + + Removes instrument configs not present in the schedule and checks that all scheduled instruments are configured. + Raises ConfigError if any scheduled instrument is missing a config. + """ + instruments_in_schedule = schedule.get_instruments() + instrument_config_map = { + InstrumentType.ARGO_FLOAT: "argo_float_config", + InstrumentType.DRIFTER: "drifter_config", + InstrumentType.XBT: "xbt_config", + InstrumentType.CTD: "ctd_config", + InstrumentType.CTD_BGC: "ctd_bgc_config", + } + # Remove configs for unused instruments + for inst_type, config_attr in instrument_config_map.items(): + if hasattr(self, config_attr) and inst_type not in instruments_in_schedule: + print( + f"{inst_type.value} configuration provided but not in schedule. Removing config." + ) + setattr(self, config_attr, None) + # Check all scheduled instruments are configured + for inst_type in instruments_in_schedule: + config_attr = instrument_config_map.get(inst_type) + if ( + not config_attr + or not hasattr(self, config_attr) + or getattr(self, config_attr) is None + ): + raise ConfigError( + f"Schedule includes instrument '{inst_type.value}', but instruments_config does not provide configuration for it." + ) + + +def _is_on_land_zero_uv(fieldset: FieldSet, waypoint: Waypoint) -> bool: + """ + Check if waypoint is on land by assuming zero velocity means land. + + :param fieldset: The fieldset to sample the velocity from. + :param waypoint: The waypoint to check. + :returns: If the waypoint is on land. + """ + return fieldset.UV.eval( + 0, + fieldset.gridset.grids[0].depth[0], + waypoint.location.lat, + waypoint.location.lon, + applyConversion=False, + ) == (0.0, 0.0) diff --git a/src/virtualship/models/schedule.py b/src/virtualship/models/schedule.py deleted file mode 100644 index 3de44f09..00000000 --- a/src/virtualship/models/schedule.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Schedule class.""" - -from __future__ import annotations - -import itertools -from datetime import datetime, timedelta -from pathlib import Path -from typing import TYPE_CHECKING - -import pydantic -import pyproj -import yaml - -from virtualship.errors import ScheduleError - -from .location import Location -from .ship_config import InstrumentType -from .space_time_region import SpaceTimeRegion - -if TYPE_CHECKING: - from parcels import FieldSet - - from virtualship.expedition.input_data import InputData - -projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") - - -class Waypoint(pydantic.BaseModel): - """A Waypoint to sail to with an optional time and an optional instrument.""" - - location: Location - time: datetime | None = None - instrument: InstrumentType | list[InstrumentType] | None = None - - @pydantic.field_serializer("instrument") - def serialize_instrument(self, instrument): - """Ensure InstrumentType is serialized as a string (or list of strings).""" - if isinstance(instrument, list): - return [inst.value for inst in instrument] - return instrument.value if instrument else None - - -class Schedule(pydantic.BaseModel): - """Schedule of the virtual ship.""" - - waypoints: list[Waypoint] - space_time_region: SpaceTimeRegion | None = None - - model_config = pydantic.ConfigDict(extra="forbid") - - def to_yaml(self, file_path: str | Path) -> None: - """ - Write schedule to yaml file. - - :param file_path: Path to the file to write to. - """ - with open(file_path, "w") as file: - yaml.dump( - self.model_dump( - by_alias=True, - ), - file, - ) - - @classmethod - def from_yaml(cls, file_path: str | Path) -> Schedule: - """ - Load schedule from yaml file. - - :param file_path: Path to the file to load from. - :returns: The schedule. - """ - with open(file_path) as file: - data = yaml.safe_load(file) - return Schedule(**data) - - def get_instruments(self) -> set[InstrumentType]: - """ - Retrieve a set of unique instruments used in the schedule. - - This method iterates through all waypoints in the schedule and collects - the instruments associated with each waypoint. It returns a set of unique - instruments, either as objects or as names. - - :raises CheckpointError: If the past waypoints in the given schedule - have been changed compared to the checkpoint. - :return: set: A set of unique instruments used in the schedule. - - """ - instruments_in_schedule = [] - for waypoint in self.waypoints: - if waypoint.instrument: - for instrument in waypoint.instrument: - if instrument: - instruments_in_schedule.append(instrument) - return set(instruments_in_schedule) - - def verify( - self, - ship_speed: float, - input_data: InputData | None, - *, - check_space_time_region: bool = False, - ignore_missing_fieldsets: bool = False, - ) -> None: - """ - Verify the feasibility and correctness of the schedule's waypoints. - - This method checks various conditions to ensure the schedule is valid: - 1. At least one waypoint is provided. - 2. The first waypoint has a specified time. - 3. Waypoint times are in ascending order. - 4. All waypoints are in water (not on land). - 5. The ship can arrive on time at each waypoint given its speed. - - :param ship_speed: The ship's speed in knots. - :param input_data: An InputData object containing fieldsets used to check if waypoints are on water. - :param check_space_time_region: whether to check for missing space_time_region. - :param ignore_missing_fieldsets: whether to ignore warning for missing field sets. - :raises PlanningError: If any of the verification checks fail, indicating infeasible or incorrect waypoints. - :raises NotImplementedError: If an instrument in the schedule is not implemented. - :return: None. The method doesn't return a value but raises exceptions if verification fails. - """ - print("\nVerifying route... ") - - if check_space_time_region and self.space_time_region is None: - raise ScheduleError( - "space_time_region not found in schedule, please define it to fetch the data." - ) - - if len(self.waypoints) == 0: - raise ScheduleError("At least one waypoint must be provided.") - - # check first waypoint has a time - if self.waypoints[0].time is None: - raise ScheduleError("First waypoint must have a specified time.") - - # check waypoint times are in ascending order - timed_waypoints = [wp for wp in self.waypoints if wp.time is not None] - checks = [ - next.time >= cur.time for cur, next in itertools.pairwise(timed_waypoints) - ] - if not all(checks): - invalid_i = [i for i, c in enumerate(checks) if c] - raise ScheduleError( - f"Waypoint(s) {', '.join(f'#{i + 1}' for i in invalid_i)}: each waypoint should be timed after all previous waypoints", - ) - - # check if all waypoints are in water - # this is done by picking an arbitrary provided fieldset and checking if UV is not zero - - # get all available fieldsets - available_fieldsets = [] - if input_data is not None: - fieldsets = [ - input_data.adcp_fieldset, - input_data.argo_float_fieldset, - input_data.ctd_fieldset, - input_data.drifter_fieldset, - input_data.ship_underwater_st_fieldset, - ] - for fs in fieldsets: - if fs is not None: - available_fieldsets.append(fs) - - # check if there are any fieldsets, else it's an error - if len(available_fieldsets) == 0: - if not ignore_missing_fieldsets: - print( - "Cannot verify because no fieldsets have been loaded. This is probably " - "because you are not using any instruments in your schedule. This is not a problem, " - "but carefully check your waypoint locations manually." - ) - - else: - # pick any - fieldset = available_fieldsets[0] - # get waypoints with 0 UV - land_waypoints = [ - (wp_i, wp) - for wp_i, wp in enumerate(self.waypoints) - if _is_on_land_zero_uv(fieldset, wp) - ] - # raise an error if there are any - if len(land_waypoints) > 0: - raise ScheduleError( - f"The following waypoints are on land: {['#' + str(wp_i) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" - ) - - # check that ship will arrive on time at each waypoint (in case no unexpected event happen) - time = self.waypoints[0].time - for wp_i, (wp, wp_next) in enumerate( - zip(self.waypoints, self.waypoints[1:], strict=False) - ): - if wp.instrument is InstrumentType.CTD: - time += timedelta(minutes=20) - - geodinv: tuple[float, float, float] = projection.inv( - wp.location.lon, - wp.location.lat, - wp_next.location.lon, - wp_next.location.lat, - ) - distance = geodinv[2] - - time_to_reach = timedelta(seconds=distance / ship_speed * 3600 / 1852) - arrival_time = time + time_to_reach - - if wp_next.time is None: - time = arrival_time - elif arrival_time > wp_next.time: - raise ScheduleError( - f"Waypoint planning is not valid: would arrive too late at waypoint number {wp_i + 2}. " - f"location: {wp_next.location} time: {wp_next.time} instrument: {wp_next.instrument}" - ) - else: - time = wp_next.time - - print("... All good to go!") - - -def _is_on_land_zero_uv(fieldset: FieldSet, waypoint: Waypoint) -> bool: - """ - Check if waypoint is on land by assuming zero velocity means land. - - :param fieldset: The fieldset to sample the velocity from. - :param waypoint: The waypoint to check. - :returns: If the waypoint is on land. - """ - return fieldset.UV.eval( - 0, - fieldset.gridset.grids[0].depth[0], - waypoint.location.lat, - waypoint.location.lon, - applyConversion=False, - ) == (0.0, 0.0) diff --git a/src/virtualship/models/ship_config.py b/src/virtualship/models/ship_config.py deleted file mode 100644 index be3ee30d..00000000 --- a/src/virtualship/models/ship_config.py +++ /dev/null @@ -1,320 +0,0 @@ -"""ShipConfig and supporting classes.""" - -from __future__ import annotations - -from datetime import timedelta -from enum import Enum -from pathlib import Path -from typing import TYPE_CHECKING - -import pydantic -import yaml - -from virtualship.errors import ConfigError -from virtualship.utils import _validate_numeric_mins_to_timedelta - -if TYPE_CHECKING: - from .schedule import Schedule - - -class InstrumentType(Enum): - """Types of the instruments.""" - - CTD = "CTD" - CTD_BGC = "CTD_BGC" - DRIFTER = "DRIFTER" - ARGO_FLOAT = "ARGO_FLOAT" - XBT = "XBT" - - -class ArgoFloatConfig(pydantic.BaseModel): - """Configuration for argos floats.""" - - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - drift_depth_meter: float = pydantic.Field(le=0.0) - vertical_speed_meter_per_second: float = pydantic.Field(lt=0.0) - cycle_days: float = pydantic.Field(gt=0.0) - drift_days: float = pydantic.Field(gt=0.0) - - -class ADCPConfig(pydantic.BaseModel): - """Configuration for ADCP instrument.""" - - max_depth_meter: float = pydantic.Field(le=0.0) - num_bins: int = pydantic.Field(gt=0.0) - period: timedelta = pydantic.Field( - serialization_alias="period_minutes", - validation_alias="period_minutes", - gt=timedelta(), - ) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("period") - def _serialize_period(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("period", mode="before") - def _validate_period(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class CTDConfig(pydantic.BaseModel): - """Configuration for CTD instrument.""" - - stationkeeping_time: timedelta = pydantic.Field( - serialization_alias="stationkeeping_time_minutes", - validation_alias="stationkeeping_time_minutes", - gt=timedelta(), - ) - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("stationkeeping_time") - def _serialize_stationkeeping_time(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("stationkeeping_time", mode="before") - def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class CTD_BGCConfig(pydantic.BaseModel): - """Configuration for CTD_BGC instrument.""" - - stationkeeping_time: timedelta = pydantic.Field( - serialization_alias="stationkeeping_time_minutes", - validation_alias="stationkeeping_time_minutes", - gt=timedelta(), - ) - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("stationkeeping_time") - def _serialize_stationkeeping_time(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("stationkeeping_time", mode="before") - def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class ShipUnderwaterSTConfig(pydantic.BaseModel): - """Configuration for underwater ST.""" - - period: timedelta = pydantic.Field( - serialization_alias="period_minutes", - validation_alias="period_minutes", - gt=timedelta(), - ) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("period") - def _serialize_period(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("period", mode="before") - def _validate_period(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class DrifterConfig(pydantic.BaseModel): - """Configuration for drifters.""" - - depth_meter: float = pydantic.Field(le=0.0) - lifetime: timedelta = pydantic.Field( - serialization_alias="lifetime_minutes", - validation_alias="lifetime_minutes", - gt=timedelta(), - ) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("lifetime") - def _serialize_lifetime(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("lifetime", mode="before") - def _validate_lifetime(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class XBTConfig(pydantic.BaseModel): - """Configuration for xbt instrument.""" - - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - fall_speed_meter_per_second: float = pydantic.Field(gt=0.0) - deceleration_coefficient: float = pydantic.Field(gt=0.0) - - -class ShipConfig(pydantic.BaseModel): - """Configuration of the virtual ship.""" - - ship_speed_knots: float = pydantic.Field(gt=0.0) - """ - Velocity of the ship in knots. - """ - - argo_float_config: ArgoFloatConfig | None = None - """ - Argo float configuration. - - If None, no argo floats can be deployed. - """ - - adcp_config: ADCPConfig | None = None - """ - ADCP configuration. - - If None, no ADCP measurements will be performed. - """ - - ctd_config: CTDConfig | None = None - """ - CTD configuration. - - If None, no CTDs can be cast. - """ - - ctd_bgc_config: CTD_BGCConfig | None = None - """ - CTD_BGC configuration. - - If None, no BGC CTDs can be cast. - """ - - ship_underwater_st_config: ShipUnderwaterSTConfig | None = None - """ - Ship underwater salinity temperature measurementconfiguration. - - If None, no ST measurements will be performed. - """ - - drifter_config: DrifterConfig | None = None - """ - Drifter configuration. - - If None, no drifters can be deployed. - """ - - xbt_config: XBTConfig | None = None - """ - XBT configuration. - - If None, no XBTs can be cast. - """ - - model_config = pydantic.ConfigDict(extra="forbid") - - def to_yaml(self, file_path: str | Path) -> None: - """ - Write config to yaml file. - - :param file_path: Path to the file to write to. - """ - with open(file_path, "w") as file: - yaml.dump(self.model_dump(by_alias=True), file) - - @classmethod - def from_yaml(cls, file_path: str | Path) -> ShipConfig: - """ - Load config from yaml file. - - :param file_path: Path to the file to load from. - :returns: The config. - """ - with open(file_path) as file: - data = yaml.safe_load(file) - return ShipConfig(**data) - - def verify(self, schedule: Schedule) -> None: - """ - Verify the ship configuration against the provided schedule. - - This function performs two main tasks: - 1. Removes instrument configurations that are not present in the schedule. - 2. Verifies that all instruments in the schedule have corresponding configurations. - - Parameters - ---------- - schedule : Schedule - The schedule object containing the planned instruments and waypoints. - - Returns - ------- - None - - Raises - ------ - ConfigError - If an instrument in the schedule does not have a corresponding configuration. - - Notes - ----- - - Prints a message if a configuration is provided for an instrument not in the schedule. - - Sets the configuration to None for instruments not in the schedule. - - Raises a ConfigError for each instrument in the schedule that lacks a configuration. - - """ - instruments_in_schedule = schedule.get_instruments() - - for instrument in [ - "ARGO_FLOAT", - "DRIFTER", - "XBT", - "CTD", - "CTD_BGC", - ]: # TODO make instrument names consistent capitals or lowercase throughout codebase - if hasattr(self, instrument.lower() + "_config") and not any( - instrument == schedule_instrument.name - for schedule_instrument in instruments_in_schedule - ): - print(f"{instrument} configuration provided but not in schedule.") - setattr(self, instrument.lower() + "_config", None) - - # verify instruments in schedule have configuration - # TODO: the ConfigError message could be improved to explain that the **schedule** file has X instrument but the **ship_config** file does not - for instrument in instruments_in_schedule: - try: - InstrumentType(instrument) - except ValueError as e: - raise NotImplementedError("Instrument not supported.") from e - - if instrument == InstrumentType.ARGO_FLOAT and ( - not hasattr(self, "argo_float_config") or self.argo_float_config is None - ): - raise ConfigError( - "Planning has a waypoint with Argo float instrument, but configuration does not configure Argo floats." - ) - if instrument == InstrumentType.CTD and ( - not hasattr(self, "ctd_config") or self.ctd_config is None - ): - raise ConfigError( - "Planning has a waypoint with CTD instrument, but configuration does not configure CTDs." - ) - if instrument == InstrumentType.CTD_BGC and ( - not hasattr(self, "ctd_bgc_config") or self.ctd_bgc_config is None - ): - raise ConfigError( - "Planning has a waypoint with CTD_BGC instrument, but configuration does not configure CTD_BGCs." - ) - if instrument == InstrumentType.DRIFTER and ( - not hasattr(self, "drifter_config") or self.drifter_config is None - ): - raise ConfigError( - "Planning has a waypoint with drifter instrument, but configuration does not configure drifters." - ) - - if instrument == InstrumentType.XBT and ( - not hasattr(self, "xbt_config") or self.xbt_config is None - ): - raise ConfigError( - "Planning has a waypoint with XBT instrument, but configuration does not configure XBT." - ) diff --git a/src/virtualship/static/expedition.yaml b/src/virtualship/static/expedition.yaml new file mode 100644 index 00000000..1a9e3922 --- /dev/null +++ b/src/virtualship/static/expedition.yaml @@ -0,0 +1,75 @@ +schedule: + space_time_region: + spatial_range: + minimum_longitude: -5 + maximum_longitude: 5 + minimum_latitude: -5 + maximum_latitude: 5 + minimum_depth: 0 + maximum_depth: 2000 + time_range: + start_time: 2023-01-01 00:00:00 + end_time: 2023-02-01 00:00:00 + waypoints: + - instrument: + - CTD + - CTD_BGC + location: + latitude: 0 + longitude: 0 + time: 2023-01-01 00:00:00 + - instrument: + - DRIFTER + - CTD + location: + latitude: 0.01 + longitude: 0.01 + time: 2023-01-01 01:00:00 + - instrument: + - ARGO_FLOAT + location: + latitude: 0.02 + longitude: 0.02 + time: 2023-01-01 02:00:00 + - instrument: + - XBT + location: + latitude: 0.03 + longitude: 0.03 + time: 2023-01-01 03:00:00 + - location: + latitude: 0.03 + longitude: 0.03 + time: 2023-01-01 03:00:00 +instruments_config: + adcp_config: + num_bins: 40 + max_depth_meter: -1000.0 + period_minutes: 5.0 + argo_float_config: + cycle_days: 10.0 + drift_days: 9.0 + drift_depth_meter: -1000.0 + max_depth_meter: -2000.0 + min_depth_meter: 0.0 + vertical_speed_meter_per_second: -0.1 + ctd_config: + max_depth_meter: -2000.0 + min_depth_meter: -11.0 + stationkeeping_time_minutes: 20.0 + ctd_bgc_config: + max_depth_meter: -2000.0 + min_depth_meter: -11.0 + stationkeeping_time_minutes: 20.0 + drifter_config: + depth_meter: 0.0 + lifetime_minutes: 60480.0 + xbt_config: + max_depth_meter: -285.0 + min_depth_meter: -2.0 + fall_speed_meter_per_second: 6.7 + deceleration_coefficient: 0.00225 + ship_underwater_st_config: + period_minutes: 5.0 +ship_config: + ship_speed_knots: 10.0 diff --git a/src/virtualship/static/schedule.yaml b/src/virtualship/static/schedule.yaml deleted file mode 100644 index 7cb39423..00000000 --- a/src/virtualship/static/schedule.yaml +++ /dev/null @@ -1,42 +0,0 @@ -space_time_region: - spatial_range: - minimum_longitude: -5 - maximum_longitude: 5 - minimum_latitude: -5 - maximum_latitude: 5 - minimum_depth: 0 - maximum_depth: 2000 - time_range: - start_time: 2023-01-01 00:00:00 - end_time: 2023-02-01 00:00:00 -waypoints: - - instrument: - - CTD - - CTD_BGC - location: - latitude: 0 - longitude: 0 - time: 2023-01-01 00:00:00 - - instrument: - - DRIFTER - - CTD - location: - latitude: 0.01 - longitude: 0.01 - time: 2023-01-01 01:00:00 - - instrument: - - ARGO_FLOAT - location: - latitude: 0.02 - longitude: 0.02 - time: 2023-01-01 02:00:00 - - instrument: - - XBT - location: - latitude: 0.03 - longitude: 0.03 - time: 2023-01-01 03:00:00 - - location: - latitude: 0.03 - longitude: 0.03 - time: 2023-01-01 03:00:00 diff --git a/src/virtualship/static/ship_config.yaml b/src/virtualship/static/ship_config.yaml deleted file mode 100644 index 34d6c6ea..00000000 --- a/src/virtualship/static/ship_config.yaml +++ /dev/null @@ -1,30 +0,0 @@ -ship_speed_knots: 10.0 -adcp_config: - num_bins: 40 - max_depth_meter: -1000.0 - period_minutes: 5.0 -argo_float_config: - cycle_days: 10.0 - drift_days: 9.0 - drift_depth_meter: -1000.0 - max_depth_meter: -2000.0 - min_depth_meter: 0.0 - vertical_speed_meter_per_second: -0.1 -ctd_config: - max_depth_meter: -2000.0 - min_depth_meter: -11.0 - stationkeeping_time_minutes: 20.0 -ctd_bgc_config: - max_depth_meter: -2000.0 - min_depth_meter: -11.0 - stationkeeping_time_minutes: 20.0 -drifter_config: - depth_meter: 0.0 - lifetime_minutes: 60480.0 -xbt_config: - max_depth_meter: -285.0 - min_depth_meter: -2.0 - fall_speed_meter_per_second: 6.7 - deceleration_coefficient: 0.00225 -ship_underwater_st_config: - period_minutes: 5.0 diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 1f334f06..0a39d035 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -8,17 +8,15 @@ from pathlib import Path from typing import TYPE_CHECKING, TextIO -from yaspin import Spinner - if TYPE_CHECKING: - from virtualship.models import Schedule, ShipConfig + from virtualship.models import Expedition import pandas as pd import yaml from pydantic import BaseModel +from yaspin import Spinner -SCHEDULE = "schedule.yaml" -SHIP_CONFIG = "ship_config.yaml" +EXPEDITION = "expedition.yaml" CHECKPOINT = "checkpoint.yaml" @@ -28,15 +26,10 @@ def load_static_file(name: str) -> str: @lru_cache(None) -def get_example_config() -> str: - """Get the example configuration file.""" - return load_static_file(SHIP_CONFIG) - - @lru_cache(None) -def get_example_schedule() -> str: - """Get the example schedule file.""" - return load_static_file(SCHEDULE) +def get_example_expedition() -> str: + """Get the example unified expedition configuration file.""" + return load_static_file(EXPEDITION) def _dump_yaml(model: BaseModel, stream: TextIO) -> str | None: @@ -121,7 +114,7 @@ def validate_coordinates(coordinates_data): def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D417 """ - Generates a YAML file with spatial and temporal information based on instrument data from MFP excel file. + Generates an expedition.yaml file with schedule information based on data from MFP excel file. The ship and instrument configurations entries in the YAML file are sourced from the static version. Parameters ---------- @@ -134,7 +127,10 @@ def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D41 4. returns the yaml information. """ + # avoid circular imports from virtualship.models import ( + Expedition, + InstrumentsConfig, Location, Schedule, SpaceTimeRegion, @@ -188,8 +184,23 @@ def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D41 space_time_region=space_time_region, ) + # extract instruments config from static + instruments_config = InstrumentsConfig.model_validate( + yaml.safe_load(get_example_expedition()).get("instruments_config") + ) + + # extract ship config from static + ship_config = yaml.safe_load(get_example_expedition()).get("ship_config") + + # combine to Expedition object + expedition = Expedition( + schedule=schedule, + instruments_config=instruments_config, + ship_config=ship_config, + ) + # Save to YAML file - schedule.to_yaml(yaml_output_path) + expedition.to_yaml(yaml_output_path) def _validate_numeric_mins_to_timedelta(value: int | float | timedelta) -> timedelta: @@ -199,26 +210,16 @@ def _validate_numeric_mins_to_timedelta(value: int | float | timedelta) -> timed return timedelta(minutes=value) -def _get_schedule(expedition_dir: Path) -> Schedule: - """Load Schedule object from yaml config file in `expedition_dir`.""" - from virtualship.models import Schedule - - file_path = expedition_dir.joinpath(SCHEDULE) - try: - return Schedule.from_yaml(file_path) - except FileNotFoundError as e: - raise FileNotFoundError(f'Schedule not found. Save it to "{file_path}".') from e - - -def _get_ship_config(expedition_dir: Path) -> ShipConfig: - from virtualship.models import ShipConfig +def _get_expedition(expedition_dir: Path) -> Expedition: + """Load Expedition object from yaml config file in `expedition_dir`.""" + from virtualship.models import Expedition - file_path = expedition_dir.joinpath(SHIP_CONFIG) + file_path = expedition_dir.joinpath(EXPEDITION) try: - return ShipConfig.from_yaml(file_path) + return Expedition.from_yaml(file_path) except FileNotFoundError as e: raise FileNotFoundError( - f'Ship config not found. Save it to "{file_path}".' + f'{EXPEDITION} not found. Save it to "{file_path}".' ) from e diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 015c3267..b8e797b7 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -4,7 +4,7 @@ from click.testing import CliRunner from virtualship.cli.commands import fetch, init -from virtualship.utils import SCHEDULE, SHIP_CONFIG +from virtualship.utils import EXPEDITION @pytest.fixture @@ -32,29 +32,16 @@ def test_init(): with runner.isolated_filesystem(): result = runner.invoke(init, ["."]) assert result.exit_code == 0 - config = Path(SHIP_CONFIG) - schedule = Path(SCHEDULE) + expedition = Path(EXPEDITION) - assert config.exists() - assert schedule.exists() + assert expedition.exists() -def test_init_existing_config(): +def test_init_existing_expedition(): runner = CliRunner() with runner.isolated_filesystem(): - config = Path(SHIP_CONFIG) - config.write_text("test") - - with pytest.raises(FileExistsError): - result = runner.invoke(init, ["."]) - raise result.exception - - -def test_init_existing_schedule(): - runner = CliRunner() - with runner.isolated_filesystem(): - schedule = Path(SCHEDULE) - schedule.write_text("test") + expedition = Path(EXPEDITION) + expedition.write_text("test") with pytest.raises(FileExistsError): result = runner.invoke(init, ["."]) diff --git a/tests/cli/test_fetch.py b/tests/cli/test_fetch.py index 856b72f6..69390733 100644 --- a/tests/cli/test_fetch.py +++ b/tests/cli/test_fetch.py @@ -16,8 +16,8 @@ hash_model, hash_to_filename, ) -from virtualship.models import Schedule, ShipConfig -from virtualship.utils import get_example_config, get_example_schedule +from virtualship.models import Expedition +from virtualship.utils import EXPEDITION, get_example_expedition @pytest.fixture @@ -32,31 +32,19 @@ def fake_download(output_filename, output_directory, **_): @pytest.fixture -def schedule(tmpdir): - out_path = tmpdir.join("schedule.yaml") +def expedition(tmpdir): + out_path = tmpdir.join(EXPEDITION) with open(out_path, "w") as file: - file.write(get_example_schedule()) + file.write(get_example_expedition()) - schedule = Schedule.from_yaml(out_path) + expedition = Expedition.from_yaml(out_path) - return schedule - - -@pytest.fixture -def ship_config(tmpdir): - out_path = tmpdir.join("ship_config.yaml") - - with open(out_path, "w") as file: - file.write(get_example_config()) - - ship_config = ShipConfig.from_yaml(out_path) - - return ship_config + return expedition @pytest.mark.usefixtures("copernicus_subset_no_download") -def test_fetch(schedule, ship_config, tmpdir): +def test_fetch(expedition, tmpdir): """Test the fetch command, but mock the download.""" _fetch(Path(tmpdir), "test", "test") diff --git a/tests/cli/test_plan.py b/tests/cli/test_plan.py index 6fef90a1..421feba0 100644 --- a/tests/cli/test_plan.py +++ b/tests/cli/test_plan.py @@ -9,7 +9,8 @@ import yaml from textual.widgets import Button, Collapsible, Input -from virtualship.cli._plan import ConfigEditor, PlanApp, ScheduleEditor +from virtualship.cli._plan import ExpeditionEditor, PlanApp +from virtualship.utils import EXPEDITION NEW_SPEED = "8.0" NEW_LAT = "0.05" @@ -33,12 +34,8 @@ async def test_UI_changes(): tmpdir = Path(tempfile.mkdtemp()) shutil.copy( - files("virtualship.static").joinpath("ship_config.yaml"), - tmpdir / "ship_config.yaml", - ) - shutil.copy( - files("virtualship.static").joinpath("schedule.yaml"), - tmpdir / "schedule.yaml", + files("virtualship.static").joinpath(EXPEDITION), + tmpdir / EXPEDITION, ) app = PlanApp(path=tmpdir) @@ -47,22 +44,23 @@ async def test_UI_changes(): await pilot.pause(0.5) plan_screen = pilot.app.screen - config_editor = plan_screen.query_one(ConfigEditor) - schedule_editor = plan_screen.query_one(ScheduleEditor) + expedition_editor = plan_screen.query_one(ExpeditionEditor) # get mock of UI notify method plan_screen.notify = MagicMock() # change ship speed - speed_collapsible = config_editor.query_one("#speed_collapsible", Collapsible) + speed_collapsible = expedition_editor.query_one( + "#speed_collapsible", Collapsible + ) if speed_collapsible.collapsed: speed_collapsible.collapsed = False await pilot.pause() - ship_speed_input = config_editor.query_one("#speed", Input) + ship_speed_input = expedition_editor.query_one("#speed", Input) await simulate_input(pilot, ship_speed_input, NEW_SPEED) # change waypoint lat/lon (e.g. first waypoint) - waypoints_collapsible = schedule_editor.query_one("#waypoints", Collapsible) + waypoints_collapsible = expedition_editor.query_one("#waypoints", Collapsible) if waypoints_collapsible.collapsed: waypoints_collapsible.collapsed = False await pilot.pause() @@ -104,11 +102,11 @@ async def test_UI_changes(): ) # verify changes to speed, lat, lon in saved YAML - ship_config_path = os.path.join(tmpdir, "ship_config.yaml") - with open(ship_config_path) as f: - saved_config = yaml.safe_load(f) + expedition_path = os.path.join(tmpdir, EXPEDITION) + with open(expedition_path) as f: + saved_expedition = yaml.safe_load(f) - assert saved_config["ship_speed_knots"] == float(NEW_SPEED) + assert saved_expedition["ship_config"]["ship_speed_knots"] == float(NEW_SPEED) # check schedule.verify() methods are working by purposefully making invalid schedule (i.e. ship speed too slow to reach waypoints) invalid_speed = "0.0001" diff --git a/tests/expedition/expedition_dir/expedition.yaml b/tests/expedition/expedition_dir/expedition.yaml new file mode 100644 index 00000000..9468028f --- /dev/null +++ b/tests/expedition/expedition_dir/expedition.yaml @@ -0,0 +1,46 @@ +schedule: + waypoints: + - instrument: + - CTD + location: + latitude: 0 + longitude: 0 + time: 2023-01-01 00:00:00 + - instrument: + - DRIFTER + - ARGO_FLOAT + location: + latitude: 0.01 + longitude: 0.01 + time: 2023-01-02 00:00:00 + - location: # empty waypoint + latitude: 0.02 + longitude: 0.01 + time: 2023-01-02 03:00:00 +instruments_config: + adcp_config: + num_bins: 40 + max_depth_meter: -1000.0 + period_minutes: 5.0 + argo_float_config: + cycle_days: 10.0 + drift_days: 9.0 + drift_depth_meter: -1000.0 + max_depth_meter: -2000.0 + min_depth_meter: 0.0 + vertical_speed_meter_per_second: -0.1 + ctd_config: + max_depth_meter: -2000.0 + min_depth_meter: -11.0 + stationkeeping_time_minutes: 20.0 + ctd_bgc_config: + max_depth_meter: -2000.0 + min_depth_meter: -11.0 + stationkeeping_time_minutes: 20.0 + drifter_config: + depth_meter: 0.0 + lifetime_minutes: 40320.0 + ship_underwater_st_config: + period_minutes: 5.0 +ship_config: + ship_speed_knots: 10.0 diff --git a/tests/expedition/expedition_dir/schedule.yaml b/tests/expedition/expedition_dir/schedule.yaml deleted file mode 100644 index 29c14ac9..00000000 --- a/tests/expedition/expedition_dir/schedule.yaml +++ /dev/null @@ -1,18 +0,0 @@ -waypoints: - - instrument: - - CTD - location: - latitude: 0 - longitude: 0 - time: 2023-01-01 00:00:00 - - instrument: - - DRIFTER - - ARGO_FLOAT - location: - latitude: 0.01 - longitude: 0.01 - time: 2023-01-02 00:00:00 - - location: # empty waypoint - latitude: 0.02 - longitude: 0.01 - time: 2023-01-02 03:00:00 diff --git a/tests/expedition/expedition_dir/ship_config.yaml b/tests/expedition/expedition_dir/ship_config.yaml deleted file mode 100644 index 1bae9d1d..00000000 --- a/tests/expedition/expedition_dir/ship_config.yaml +++ /dev/null @@ -1,25 +0,0 @@ -ship_speed_knots: 10.0 -adcp_config: - num_bins: 40 - max_depth_meter: -1000.0 - period_minutes: 5.0 -argo_float_config: - cycle_days: 10.0 - drift_days: 9.0 - drift_depth_meter: -1000.0 - max_depth_meter: -2000.0 - min_depth_meter: 0.0 - vertical_speed_meter_per_second: -0.1 -ctd_config: - max_depth_meter: -2000.0 - min_depth_meter: -11.0 - stationkeeping_time_minutes: 20.0 -ctd_bgc_config: - max_depth_meter: -2000.0 - min_depth_meter: -11.0 - stationkeeping_time_minutes: 20.0 -drifter_config: - depth_meter: 0.0 - lifetime_minutes: 40320.0 -ship_underwater_st_config: - period_minutes: 5.0 diff --git a/tests/expedition/test_expedition.py b/tests/expedition/test_expedition.py new file mode 100644 index 00000000..a4643e03 --- /dev/null +++ b/tests/expedition/test_expedition.py @@ -0,0 +1,277 @@ +from datetime import datetime, timedelta +from pathlib import Path + +import pyproj +import pytest + +from virtualship.errors import ConfigError, ScheduleError +from virtualship.expedition.do_expedition import _load_input_data +from virtualship.models import Expedition, Location, Schedule, Waypoint +from virtualship.utils import EXPEDITION, _get_expedition, get_example_expedition + +projection = pyproj.Geod(ellps="WGS84") + +expedition_dir = Path("expedition_dir") + + +def test_import_export_expedition(tmpdir) -> None: + out_path = tmpdir.join(EXPEDITION) + + # arbitrary time for testing + base_time = datetime.strptime("1950-01-01", "%Y-%m-%d") + + schedule = Schedule( + waypoints=[ + Waypoint(location=Location(0, 0), time=base_time, instrument=None), + Waypoint( + location=Location(1, 1), + time=base_time + timedelta(hours=1), + instrument=None, + ), + ] + ) + get_expedition = _get_expedition(expedition_dir) + expedition = Expedition( + schedule=schedule, + instruments_config=get_expedition.instruments_config, + ship_config=get_expedition.ship_config, + ) + expedition.to_yaml(out_path) + + expedition2 = Expedition.from_yaml(out_path) + assert expedition == expedition2 + + +def test_verify_schedule() -> None: + schedule = Schedule( + waypoints=[ + Waypoint(location=Location(0, 0), time=datetime(2022, 1, 1, 1, 0, 0)), + Waypoint(location=Location(1, 0), time=datetime(2022, 1, 2, 1, 0, 0)), + ] + ) + + ship_speed_knots = _get_expedition(expedition_dir).ship_config.ship_speed_knots + + schedule.verify(ship_speed_knots, None) + + +def test_get_instruments() -> None: + schedule = Schedule( + waypoints=[ + Waypoint(location=Location(0, 0), instrument=["CTD"]), + Waypoint(location=Location(1, 0), instrument=["XBT", "ARGO_FLOAT"]), + Waypoint(location=Location(1, 0), instrument=["CTD"]), + ] + ) + + assert set(instrument.name for instrument in schedule.get_instruments()) == { + "CTD", + "XBT", + "ARGO_FLOAT", + } + + +@pytest.mark.parametrize( + "schedule,check_space_time_region,error,match", + [ + pytest.param( + Schedule(waypoints=[]), + False, + ScheduleError, + "At least one waypoint must be provided.", + id="NoWaypoints", + ), + pytest.param( + Schedule( + waypoints=[ + Waypoint(location=Location(0, 0)), + Waypoint( + location=Location(1, 0), time=datetime(2022, 1, 1, 1, 0, 0) + ), + ] + ), + False, + ScheduleError, + "First waypoint must have a specified time.", + id="FirstWaypointHasTime", + ), + pytest.param( + Schedule( + waypoints=[ + Waypoint( + location=Location(0, 0), time=datetime(2022, 1, 2, 1, 0, 0) + ), + Waypoint(location=Location(0, 0)), + Waypoint( + location=Location(1, 0), time=datetime(2022, 1, 1, 1, 0, 0) + ), + ] + ), + False, + ScheduleError, + "Waypoint\\(s\\) : each waypoint should be timed after all previous waypoints", + id="SequentialWaypoints", + ), + pytest.param( + Schedule( + waypoints=[ + Waypoint( + location=Location(0, 0), time=datetime(2022, 1, 1, 1, 0, 0) + ), + Waypoint( + location=Location(1, 0), time=datetime(2022, 1, 1, 1, 1, 0) + ), + ] + ), + False, + ScheduleError, + "Waypoint planning is not valid: would arrive too late at waypoint number 2...", + id="NotEnoughTime", + ), + pytest.param( + Schedule( + waypoints=[ + Waypoint( + location=Location(0, 0), time=datetime(2022, 1, 1, 1, 0, 0) + ), + Waypoint( + location=Location(1, 0), time=datetime(2022, 1, 2, 1, 1, 0) + ), + ] + ), + True, + ScheduleError, + "space_time_region not found in schedule, please define it to fetch the data.", + id="NoSpaceTimeRegion", + ), + ], +) +def test_verify_schedule_errors( + schedule: Schedule, check_space_time_region: bool, error, match +) -> None: + expedition = _get_expedition(expedition_dir) + input_data = _load_input_data( + expedition_dir, + expedition, + input_data=Path("expedition_dir/input_data"), + ) + + with pytest.raises(error, match=match): + schedule.verify( + expedition.ship_config.ship_speed_knots, + input_data, + check_space_time_region=check_space_time_region, + ) + + +@pytest.fixture +def schedule(tmp_file): + with open(tmp_file, "w") as file: + file.write(get_example_expedition()) + return Expedition.from_yaml(tmp_file).schedule + + +@pytest.fixture +def schedule_no_xbt(schedule): + for waypoint in schedule.waypoints: + if waypoint.instrument and any( + instrument.name == "XBT" for instrument in waypoint.instrument + ): + waypoint.instrument = [ + instrument + for instrument in waypoint.instrument + if instrument.name != "XBT" + ] + + return schedule + + +@pytest.fixture +def instruments_config(tmp_file): + with open(tmp_file, "w") as file: + file.write(get_example_expedition()) + return Expedition.from_yaml(tmp_file).instruments_config + + +@pytest.fixture +def instruments_config_no_xbt(instruments_config): + delattr(instruments_config, "xbt_config") + return instruments_config + + +@pytest.fixture +def instruments_config_no_ctd(instruments_config): + delattr(instruments_config, "ctd_config") + return instruments_config + + +@pytest.fixture +def instruments_config_no_ctd_bgc(instruments_config): + delattr(instruments_config, "ctd_bgc_config") + return instruments_config + + +@pytest.fixture +def instruments_config_no_argo_float(instruments_config): + delattr(instruments_config, "argo_float_config") + return instruments_config + + +@pytest.fixture +def instruments_config_no_drifter(instruments_config): + delattr(instruments_config, "drifter_config") + return instruments_config + + +def test_verify_instruments_config(instruments_config, schedule) -> None: + instruments_config.verify(schedule) + + +def test_verify_instruments_config_no_instrument( + instruments_config, schedule_no_xbt +) -> None: + instruments_config.verify(schedule_no_xbt) + + +@pytest.mark.parametrize( + "instruments_config_fixture,error,match", + [ + pytest.param( + "instruments_config_no_xbt", + ConfigError, + "Schedule includes instrument 'XBT', but instruments_config does not provide configuration for it.", + id="ShipConfigNoXBT", + ), + pytest.param( + "instruments_config_no_ctd", + ConfigError, + "Schedule includes instrument 'CTD', but instruments_config does not provide configuration for it.", + id="ShipConfigNoCTD", + ), + pytest.param( + "instruments_config_no_ctd_bgc", + ConfigError, + "Schedule includes instrument 'CTD_BGC', but instruments_config does not provide configuration for it.", + id="ShipConfigNoCTD_BGC", + ), + pytest.param( + "instruments_config_no_argo_float", + ConfigError, + "Schedule includes instrument 'ARGO_FLOAT', but instruments_config does not provide configuration for it.", + id="ShipConfigNoARGO_FLOAT", + ), + pytest.param( + "instruments_config_no_drifter", + ConfigError, + "Schedule includes instrument 'DRIFTER', but instruments_config does not provide configuration for it.", + id="ShipConfigNoDRIFTER", + ), + ], +) +def test_verify_instruments_config_errors( + request, schedule, instruments_config_fixture, error, match +) -> None: + instruments_config = request.getfixturevalue(instruments_config_fixture) + + with pytest.raises(error, match=match): + instruments_config.verify(schedule) diff --git a/tests/expedition/test_schedule.py b/tests/expedition/test_schedule.py deleted file mode 100644 index f4a8532e..00000000 --- a/tests/expedition/test_schedule.py +++ /dev/null @@ -1,160 +0,0 @@ -from datetime import datetime, timedelta -from pathlib import Path - -import pyproj -import pytest - -from virtualship.errors import ScheduleError -from virtualship.expedition.do_expedition import _load_input_data -from virtualship.models import Location, Schedule, Waypoint -from virtualship.utils import _get_ship_config - -projection = pyproj.Geod(ellps="WGS84") - -expedition_dir = Path("expedition_dir") - - -def test_import_export_schedule(tmpdir) -> None: - out_path = tmpdir.join("schedule.yaml") - - # arbitrary time for testing - base_time = datetime.strptime("1950-01-01", "%Y-%m-%d") - - schedule = Schedule( - waypoints=[ - Waypoint(location=Location(0, 0), time=base_time, instrument=None), - Waypoint( - location=Location(1, 1), - time=base_time + timedelta(hours=1), - instrument=None, - ), - ] - ) - schedule.to_yaml(out_path) - - schedule2 = Schedule.from_yaml(out_path) - assert schedule == schedule2 - - -def test_verify_schedule() -> None: - schedule = Schedule( - waypoints=[ - Waypoint(location=Location(0, 0), time=datetime(2022, 1, 1, 1, 0, 0)), - Waypoint(location=Location(1, 0), time=datetime(2022, 1, 2, 1, 0, 0)), - ] - ) - - ship_config = _get_ship_config(expedition_dir) - - schedule.verify(ship_config.ship_speed_knots, None) - - -def test_get_instruments() -> None: - schedule = Schedule( - waypoints=[ - Waypoint(location=Location(0, 0), instrument=["CTD"]), - Waypoint(location=Location(1, 0), instrument=["XBT", "ARGO_FLOAT"]), - Waypoint(location=Location(1, 0), instrument=["CTD"]), - ] - ) - - assert set(instrument.name for instrument in schedule.get_instruments()) == { - "CTD", - "XBT", - "ARGO_FLOAT", - } - - -@pytest.mark.parametrize( - "schedule,check_space_time_region,error,match", - [ - pytest.param( - Schedule(waypoints=[]), - False, - ScheduleError, - "At least one waypoint must be provided.", - id="NoWaypoints", - ), - pytest.param( - Schedule( - waypoints=[ - Waypoint(location=Location(0, 0)), - Waypoint( - location=Location(1, 0), time=datetime(2022, 1, 1, 1, 0, 0) - ), - ] - ), - False, - ScheduleError, - "First waypoint must have a specified time.", - id="FirstWaypointHasTime", - ), - pytest.param( - Schedule( - waypoints=[ - Waypoint( - location=Location(0, 0), time=datetime(2022, 1, 2, 1, 0, 0) - ), - Waypoint(location=Location(0, 0)), - Waypoint( - location=Location(1, 0), time=datetime(2022, 1, 1, 1, 0, 0) - ), - ] - ), - False, - ScheduleError, - "Waypoint\\(s\\) : each waypoint should be timed after all previous waypoints", - id="SequentialWaypoints", - ), - pytest.param( - Schedule( - waypoints=[ - Waypoint( - location=Location(0, 0), time=datetime(2022, 1, 1, 1, 0, 0) - ), - Waypoint( - location=Location(1, 0), time=datetime(2022, 1, 1, 1, 1, 0) - ), - ] - ), - False, - ScheduleError, - "Waypoint planning is not valid: would arrive too late at waypoint number 2...", - id="NotEnoughTime", - ), - pytest.param( - Schedule( - waypoints=[ - Waypoint( - location=Location(0, 0), time=datetime(2022, 1, 1, 1, 0, 0) - ), - Waypoint( - location=Location(1, 0), time=datetime(2022, 1, 2, 1, 1, 0) - ), - ] - ), - True, - ScheduleError, - "space_time_region not found in schedule, please define it to fetch the data.", - id="NoSpaceTimeRegion", - ), - ], -) -def test_verify_schedule_errors( - schedule: Schedule, check_space_time_region: bool, error, match -) -> None: - ship_config = _get_ship_config(expedition_dir) - - input_data = _load_input_data( - expedition_dir, - schedule, - ship_config, - input_data=Path("expedition_dir/input_data"), - ) - - with pytest.raises(error, match=match): - schedule.verify( - ship_config.ship_speed_knots, - input_data, - check_space_time_region=check_space_time_region, - ) diff --git a/tests/expedition/test_ship_config.py b/tests/expedition/test_ship_config.py deleted file mode 100644 index 6444e985..00000000 --- a/tests/expedition/test_ship_config.py +++ /dev/null @@ -1,126 +0,0 @@ -from pathlib import Path - -import pytest - -from virtualship.errors import ConfigError -from virtualship.models import Schedule, ShipConfig -from virtualship.utils import get_example_config, get_example_schedule - -expedition_dir = Path("expedition_dir") - - -@pytest.fixture -def schedule(tmp_file): - with open(tmp_file, "w") as file: - file.write(get_example_schedule()) - return Schedule.from_yaml(tmp_file) - - -@pytest.fixture -def schedule_no_xbt(schedule): - for waypoint in schedule.waypoints: - if waypoint.instrument and any( - instrument.name == "XBT" for instrument in waypoint.instrument - ): - waypoint.instrument = [ - instrument - for instrument in waypoint.instrument - if instrument.name != "XBT" - ] - - return schedule - - -@pytest.fixture -def ship_config(tmp_file): - with open(tmp_file, "w") as file: - file.write(get_example_config()) - return ShipConfig.from_yaml(tmp_file) - - -@pytest.fixture -def ship_config_no_xbt(ship_config): - delattr(ship_config, "xbt_config") - return ship_config - - -@pytest.fixture -def ship_config_no_ctd(ship_config): - delattr(ship_config, "ctd_config") - return ship_config - - -@pytest.fixture -def ship_config_no_ctd_bgc(ship_config): - delattr(ship_config, "ctd_bgc_config") - return ship_config - - -@pytest.fixture -def ship_config_no_argo_float(ship_config): - delattr(ship_config, "argo_float_config") - return ship_config - - -@pytest.fixture -def ship_config_no_drifter(ship_config): - delattr(ship_config, "drifter_config") - return ship_config - - -def test_import_export_ship_config(ship_config, tmp_file) -> None: - ship_config.to_yaml(tmp_file) - ship_config_2 = ShipConfig.from_yaml(tmp_file) - assert ship_config == ship_config_2 - - -def test_verify_ship_config(ship_config, schedule) -> None: - ship_config.verify(schedule) - - -def test_verify_ship_config_no_instrument(ship_config, schedule_no_xbt) -> None: - ship_config.verify(schedule_no_xbt) - - -@pytest.mark.parametrize( - "ship_config_fixture,error,match", - [ - pytest.param( - "ship_config_no_xbt", - ConfigError, - "Planning has a waypoint with XBT instrument, but configuration does not configure XBT.", - id="ShipConfigNoXBT", - ), - pytest.param( - "ship_config_no_ctd", - ConfigError, - "Planning has a waypoint with CTD instrument, but configuration does not configure CTD.", - id="ShipConfigNoCTD", - ), - pytest.param( - "ship_config_no_ctd_bgc", - ConfigError, - "Planning has a waypoint with CTD_BGC instrument, but configuration does not configure CTD_BGCs.", - id="ShipConfigNoCTD_BGC", - ), - pytest.param( - "ship_config_no_argo_float", - ConfigError, - "Planning has a waypoint with Argo float instrument, but configuration does not configure Argo floats.", - id="ShipConfigNoARGO_FLOAT", - ), - pytest.param( - "ship_config_no_drifter", - ConfigError, - "Planning has a waypoint with drifter instrument, but configuration does not configure drifters.", - id="ShipConfigNoDRIFTER", - ), - ], -) -def test_verify_ship_config_errors( - request, schedule, ship_config_fixture, error, match -) -> None: - ship_config = request.getfixturevalue(ship_config_fixture) - - with pytest.raises(error, match=match): - ship_config.verify(schedule) diff --git a/tests/expedition/test_simulate_schedule.py b/tests/expedition/test_simulate_schedule.py index 9eecd73d..bad8c9ad 100644 --- a/tests/expedition/test_simulate_schedule.py +++ b/tests/expedition/test_simulate_schedule.py @@ -7,7 +7,7 @@ ScheduleProblem, simulate_schedule, ) -from virtualship.models import Location, Schedule, ShipConfig, Waypoint +from virtualship.models import Expedition, Location, Schedule, Waypoint def test_simulate_schedule_feasible() -> None: @@ -15,16 +15,16 @@ def test_simulate_schedule_feasible() -> None: base_time = datetime.strptime("2022-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S") projection = pyproj.Geod(ellps="WGS84") - ship_config = ShipConfig.from_yaml("expedition_dir/ship_config.yaml") - ship_config.ship_speed_knots = 10.0 - schedule = Schedule( + expedition = Expedition.from_yaml("expedition_dir/expedition.yaml") + expedition.ship_config.ship_speed_knots = 10.0 + expedition.schedule = Schedule( waypoints=[ Waypoint(location=Location(0, 0), time=base_time), Waypoint(location=Location(0.01, 0), time=base_time + timedelta(days=1)), ] ) - result = simulate_schedule(projection, ship_config, schedule) + result = simulate_schedule(projection, expedition) assert isinstance(result, ScheduleOk) @@ -34,23 +34,28 @@ def test_simulate_schedule_too_far() -> None: base_time = datetime.strptime("2022-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S") projection = pyproj.Geod(ellps="WGS84") - ship_config = ShipConfig.from_yaml("expedition_dir/ship_config.yaml") - schedule = Schedule( + expedition = Expedition.from_yaml("expedition_dir/expedition.yaml") + expedition.ship_config.ship_speed_knots = 10.0 + expedition.schedule = Schedule( waypoints=[ Waypoint(location=Location(0, 0), time=base_time), Waypoint(location=Location(1.0, 0), time=base_time + timedelta(minutes=1)), ] ) - result = simulate_schedule(projection, ship_config, schedule) + result = simulate_schedule(projection, expedition) assert isinstance(result, ScheduleProblem) def test_time_in_minutes_in_ship_schedule() -> None: """Test whether the pydantic serializer picks up the time *in minutes* in the ship schedule.""" - ship_config = ShipConfig.from_yaml("expedition_dir/ship_config.yaml") - assert ship_config.adcp_config.period == timedelta(minutes=5) - assert ship_config.ctd_config.stationkeeping_time == timedelta(minutes=20) - assert ship_config.ctd_bgc_config.stationkeeping_time == timedelta(minutes=20) - assert ship_config.ship_underwater_st_config.period == timedelta(minutes=5) + instruments_config = Expedition.from_yaml( + "expedition_dir/expedition.yaml" + ).instruments_config + assert instruments_config.adcp_config.period == timedelta(minutes=5) + assert instruments_config.ctd_config.stationkeeping_time == timedelta(minutes=20) + assert instruments_config.ctd_bgc_config.stationkeeping_time == timedelta( + minutes=20 + ) + assert instruments_config.ship_underwater_st_config.period == timedelta(minutes=5) diff --git a/tests/test_mfp_to_yaml.py b/tests/test_mfp_to_yaml.py index d242d30a..4eab16c2 100644 --- a/tests/test_mfp_to_yaml.py +++ b/tests/test_mfp_to_yaml.py @@ -3,7 +3,7 @@ import pandas as pd import pytest -from virtualship.models import Schedule +from virtualship.models import Expedition from virtualship.utils import mfp_to_yaml @@ -88,7 +88,7 @@ def test_mfp_to_yaml_success(request, fixture_name, tmp_path): """Test that mfp_to_yaml correctly processes a valid MFP file.""" valid_mfp_file = request.getfixturevalue(fixture_name) - yaml_output_path = tmp_path / "schedule.yaml" + yaml_output_path = tmp_path / "expedition.yaml" # Run function (No need to mock open() for YAML, real file is created) mfp_to_yaml(valid_mfp_file, yaml_output_path) @@ -97,9 +97,9 @@ def test_mfp_to_yaml_success(request, fixture_name, tmp_path): assert yaml_output_path.exists() # Load YAML and validate contents - data = Schedule.from_yaml(yaml_output_path) + data = Expedition.from_yaml(yaml_output_path) - assert len(data.waypoints) == 3 + assert len(data.schedule.waypoints) == 3 @pytest.mark.parametrize( @@ -138,7 +138,7 @@ def test_mfp_to_yaml_exceptions(request, fixture_name, error, match, tmp_path): """Test that mfp_to_yaml raises an error when input file is not valid.""" fixture = request.getfixturevalue(fixture_name) - yaml_output_path = tmp_path / "schedule.yaml" + yaml_output_path = tmp_path / "expedition.yaml" with pytest.raises(error, match=match): mfp_to_yaml(fixture, yaml_output_path) @@ -146,7 +146,7 @@ def test_mfp_to_yaml_exceptions(request, fixture_name, error, match, tmp_path): def test_mfp_to_yaml_extra_headers(unexpected_header_mfp_file, tmp_path): """Test that mfp_to_yaml prints a warning when extra columns are found.""" - yaml_output_path = tmp_path / "schedule.yaml" + yaml_output_path = tmp_path / "expedition.yaml" with pytest.warns(UserWarning, match="Found additional unexpected columns.*"): mfp_to_yaml(unexpected_header_mfp_file, yaml_output_path) diff --git a/tests/test_utils.py b/tests/test_utils.py index 4c6db8fc..0dcebd79 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,26 +1,14 @@ -from virtualship.models import Schedule, ShipConfig -from virtualship.utils import get_example_config, get_example_schedule +from virtualship.models import Expedition +from virtualship.utils import get_example_expedition -def test_get_example_config(): - assert len(get_example_config()) > 0 +def test_get_example_expedition(): + assert len(get_example_expedition()) > 0 -def test_get_example_schedule(): - assert len(get_example_schedule()) > 0 - - -def test_valid_example_config(tmp_path): - path = tmp_path / "test.yaml" - with open(path, "w") as file: - file.write(get_example_config()) - - ShipConfig.from_yaml(path) - - -def test_valid_example_schedule(tmp_path): +def test_valid_example_expedition(tmp_path): path = tmp_path / "test.yaml" with open(path, "w") as file: - file.write(get_example_schedule()) + file.write(get_example_expedition()) - Schedule.from_yaml(path) + Expedition.from_yaml(path) From 2240b899913776a9c1ff8264ce233fc8870905a5 Mon Sep 17 00:00:00 2001 From: Nick Hodgskin <36369090+VeckoTheGecko@users.noreply.github.com> Date: Wed, 8 Oct 2025 14:43:45 +0200 Subject: [PATCH 02/97] Update link to website (#215) --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 0210f2f5..d1cd43fc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,7 +8,7 @@ Home user-guide/index api/index contributing/index -VirtualShip Website +VirtualShip Website ``` ```{include} ../README.md From 63cbda5f756384311c32cbd03a2ec77d975cebc8 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 18 Aug 2025 21:01:36 +0100 Subject: [PATCH 03/97] first refactoring step, parent instrument classes --- src/virtualship/cli/_fetch.py | 11 ++ src/virtualship/instruments/adcp.py | 78 -------- src/virtualship/instruments/argo_float.py | 186 ------------------ src/virtualship/instruments/ctd.py | 164 ++++----------- src/virtualship/instruments/drifter.py | 113 ----------- .../instruments/ship_underwater_st.py | 76 ------- src/virtualship/instruments/xbt.py | 141 ------------- src/virtualship/models/instruments.py | 82 ++++++++ 8 files changed, 135 insertions(+), 716 deletions(-) delete mode 100644 src/virtualship/instruments/adcp.py delete mode 100644 src/virtualship/instruments/argo_float.py delete mode 100644 src/virtualship/instruments/drifter.py delete mode 100644 src/virtualship/instruments/ship_underwater_st.py delete mode 100644 src/virtualship/instruments/xbt.py create mode 100644 src/virtualship/models/instruments.py diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 60008304..bf35fe2f 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -86,6 +86,17 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None ) shutil.copyfile(path / EXPEDITION, download_folder / EXPEDITION) + #! + #### TODO + # ++ new logic here where iterates (?) through available instruments and determines whether download is required: + # ++ by conditions of: + # 1) whether it's in the schedule (and from this be able to call the right classes from the instruments directory?) and + #! 2) is there a clever way of not unnecessarily duplicating data downloads if instruments use the same?! + # (try with a version first where does them all in tow and then try and optimise...?) + + #! + ## TODO: move to generic bathymetry download which is done for all expeditions + if ( ( {"XBT", "CTD", "CDT_BGC", "SHIP_UNDERWATER_ST"} diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py deleted file mode 100644 index af2c285e..00000000 --- a/src/virtualship/instruments/adcp.py +++ /dev/null @@ -1,78 +0,0 @@ -"""ADCP instrument.""" - -from pathlib import Path - -import numpy as np -from parcels import FieldSet, ParticleSet, ScipyParticle, Variable - -from virtualship.models import Spacetime - -# we specifically use ScipyParticle because we have many small calls to execute -# there is some overhead with JITParticle and this ends up being significantly faster -_ADCPParticle = ScipyParticle.add_variables( - [ - Variable("U", dtype=np.float32, initial=np.nan), - Variable("V", dtype=np.float32, initial=np.nan), - ] -) - - -def _sample_velocity(particle, fieldset, time): - particle.U, particle.V = fieldset.UV.eval( - time, particle.depth, particle.lat, particle.lon, applyConversion=False - ) - - -def simulate_adcp( - fieldset: FieldSet, - out_path: str | Path, - max_depth: float, - min_depth: float, - num_bins: int, - sample_points: list[Spacetime], -) -> None: - """ - Use Parcels to simulate an ADCP in a fieldset. - - :param fieldset: The fieldset to simulate the ADCP in. - :param out_path: The path to write the results to. - :param max_depth: Maximum depth the ADCP can measure. - :param min_depth: Minimum depth the ADCP can measure. - :param num_bins: How many samples to take in the complete range between max_depth and min_depth. - :param sample_points: The places and times to sample at. - """ - sample_points.sort(key=lambda p: p.time) - - bins = np.linspace(max_depth, min_depth, num_bins) - num_particles = len(bins) - particleset = ParticleSet.from_list( - fieldset=fieldset, - pclass=_ADCPParticle, - lon=np.full( - num_particles, 0.0 - ), # initial lat/lon are irrelevant and will be overruled later. - lat=np.full(num_particles, 0.0), - depth=bins, - time=0, # same for time - ) - - # define output file for the simulation - # outputdt set to infinite as we just want to write at the end of every call to 'execute' - out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) - - for point in sample_points: - particleset.lon_nextloop[:] = point.location.lon - particleset.lat_nextloop[:] = point.location.lat - particleset.time_nextloop[:] = fieldset.time_origin.reltime( - np.datetime64(point.time) - ) - - # perform one step using the particleset - # dt and runtime are set so exactly one step is made. - particleset.execute( - [_sample_velocity], - dt=1, - runtime=1, - verbose_progress=False, - output_file=out_file, - ) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py deleted file mode 100644 index d0976367..00000000 --- a/src/virtualship/instruments/argo_float.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Argo float instrument.""" - -import math -from dataclasses import dataclass -from datetime import datetime, timedelta -from pathlib import Path - -import numpy as np -from parcels import ( - AdvectionRK4, - FieldSet, - JITParticle, - ParticleSet, - StatusCode, - Variable, -) - -from virtualship.models import Spacetime - - -@dataclass -class ArgoFloat: - """Configuration for a single Argo float.""" - - spacetime: Spacetime - min_depth: float - max_depth: float - drift_depth: float - vertical_speed: float - cycle_days: float - drift_days: float - - -_ArgoParticle = JITParticle.add_variables( - [ - Variable("cycle_phase", dtype=np.int32, initial=0.0), - Variable("cycle_age", dtype=np.float32, initial=0.0), - Variable("drift_age", dtype=np.float32, initial=0.0), - Variable("salinity", dtype=np.float32, initial=np.nan), - Variable("temperature", dtype=np.float32, initial=np.nan), - Variable("min_depth", dtype=np.float32), - Variable("max_depth", dtype=np.float32), - Variable("drift_depth", dtype=np.float32), - Variable("vertical_speed", dtype=np.float32), - Variable("cycle_days", dtype=np.int32), - Variable("drift_days", dtype=np.int32), - ] -) - - -def _argo_float_vertical_movement(particle, fieldset, time): - if particle.cycle_phase == 0: - # Phase 0: Sinking with vertical_speed until depth is drift_depth - particle_ddepth += ( # noqa Parcels defines particle_* variables, which code checkers cannot know. - particle.vertical_speed * particle.dt - ) - if particle.depth + particle_ddepth <= particle.drift_depth: - particle_ddepth = particle.drift_depth - particle.depth - particle.cycle_phase = 1 - - elif particle.cycle_phase == 1: - # Phase 1: Drifting at depth for drifttime seconds - particle.drift_age += particle.dt - if particle.drift_age >= particle.drift_days * 86400: - particle.drift_age = 0 # reset drift_age for next cycle - particle.cycle_phase = 2 - - elif particle.cycle_phase == 2: - # Phase 2: Sinking further to max_depth - particle_ddepth += particle.vertical_speed * particle.dt - if particle.depth + particle_ddepth <= particle.max_depth: - particle_ddepth = particle.max_depth - particle.depth - particle.cycle_phase = 3 - - elif particle.cycle_phase == 3: - # Phase 3: Rising with vertical_speed until at surface - particle_ddepth -= particle.vertical_speed * particle.dt - particle.cycle_age += ( - particle.dt - ) # solve issue of not updating cycle_age during ascent - if particle.depth + particle_ddepth >= particle.min_depth: - particle_ddepth = particle.min_depth - particle.depth - particle.temperature = ( - math.nan - ) # reset temperature to NaN at end of sampling cycle - particle.salinity = math.nan # idem - particle.cycle_phase = 4 - else: - particle.temperature = fieldset.T[ - time, particle.depth, particle.lat, particle.lon - ] - particle.salinity = fieldset.S[ - time, particle.depth, particle.lat, particle.lon - ] - - elif particle.cycle_phase == 4: - # Phase 4: Transmitting at surface until cycletime is reached - if particle.cycle_age > particle.cycle_days * 86400: - particle.cycle_phase = 0 - particle.cycle_age = 0 - - if particle.state == StatusCode.Evaluate: - particle.cycle_age += particle.dt # update cycle_age - - -def _keep_at_surface(particle, fieldset, time): - # Prevent error when float reaches surface - if particle.state == StatusCode.ErrorThroughSurface: - particle.depth = particle.min_depth - particle.state = StatusCode.Success - - -def _check_error(particle, fieldset, time): - if particle.state >= 50: # This captures all Errors - particle.delete() - - -def simulate_argo_floats( - fieldset: FieldSet, - out_path: str | Path, - argo_floats: list[ArgoFloat], - outputdt: timedelta, - endtime: datetime | None, -) -> None: - """ - Use Parcels to simulate a set of Argo floats in a fieldset. - - :param fieldset: The fieldset to simulate the Argo floats in. - :param out_path: The path to write the results to. - :param argo_floats: A list of Argo floats to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :param endtime: Stop at this time, or if None, continue until the end of the fieldset. - """ - DT = 10.0 # dt of Argo float simulation integrator - - if len(argo_floats) == 0: - print( - "No Argo floats provided. Parcels currently crashes when providing an empty particle set, so no argo floats simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return - - # define parcel particles - argo_float_particleset = ParticleSet( - fieldset=fieldset, - pclass=_ArgoParticle, - lat=[argo.spacetime.location.lat for argo in argo_floats], - lon=[argo.spacetime.location.lon for argo in argo_floats], - depth=[argo.min_depth for argo in argo_floats], - time=[argo.spacetime.time for argo in argo_floats], - min_depth=[argo.min_depth for argo in argo_floats], - max_depth=[argo.max_depth for argo in argo_floats], - drift_depth=[argo.drift_depth for argo in argo_floats], - vertical_speed=[argo.vertical_speed for argo in argo_floats], - cycle_days=[argo.cycle_days for argo in argo_floats], - drift_days=[argo.drift_days for argo in argo_floats], - ) - - # define output file for the simulation - out_file = argo_float_particleset.ParticleFile( - name=out_path, outputdt=outputdt, chunks=[len(argo_float_particleset), 100] - ) - - # get earliest between fieldset end time and provide end time - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - if endtime is None: - actual_endtime = fieldset_endtime - elif endtime > fieldset_endtime: - print("WARN: Requested end time later than fieldset end time.") - actual_endtime = fieldset_endtime - else: - actual_endtime = np.timedelta64(endtime) - - # execute simulation - argo_float_particleset.execute( - [ - _argo_float_vertical_movement, - AdvectionRK4, - _keep_at_surface, - _check_error, - ], - endtime=actual_endtime, - dt=DT, - output_file=out_file, - verbose_progress=True, - ) diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 41185007..0d7294fb 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -1,137 +1,57 @@ -"""CTD instrument.""" - from dataclasses import dataclass -from datetime import timedelta from pathlib import Path -import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable +from virtualship.models import Spacetime, instruments -from virtualship.models import Spacetime +MYINSTRUMENT = "CTD" @dataclass class CTD: - """Configuration for a single CTD.""" + """CTD configuration.""" spacetime: Spacetime min_depth: float max_depth: float -_CTDParticle = JITParticle.add_variables( - [ - Variable("salinity", dtype=np.float32, initial=np.nan), - Variable("temperature", dtype=np.float32, initial=np.nan), - Variable("raising", dtype=np.int8, initial=0.0), # bool. 0 is False, 1 is True. - Variable("max_depth", dtype=np.float32), - Variable("min_depth", dtype=np.float32), - Variable("winch_speed", dtype=np.float32), - ] -) - - -def _sample_temperature(particle, fieldset, time): - particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] - - -def _sample_salinity(particle, fieldset, time): - particle.salinity = fieldset.S[time, particle.depth, particle.lat, particle.lon] - - -def _ctd_cast(particle, fieldset, time): - # lowering - if particle.raising == 0: - particle_ddepth = -particle.winch_speed * particle.dt - if particle.depth + particle_ddepth < particle.max_depth: - particle.raising = 1 - particle_ddepth = -particle_ddepth - # raising - else: - particle_ddepth = particle.winch_speed * particle.dt - if particle.depth + particle_ddepth > particle.min_depth: - particle.delete() - - -def simulate_ctd( - fieldset: FieldSet, - out_path: str | Path, - ctds: list[CTD], - outputdt: timedelta, -) -> None: - """ - Use Parcels to simulate a set of CTDs in a fieldset. - - :param fieldset: The fieldset to simulate the CTDs in. - :param out_path: The path to write the results to. - :param ctds: A list of CTDs to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :raises ValueError: Whenever provided CTDs, fieldset, are not compatible with this function. - """ - WINCH_SPEED = 1.0 # sink and rise speed in m/s - DT = 10.0 # dt of CTD simulation integrator - - if len(ctds) == 0: - print( - "No CTDs provided. Parcels currently crashes when providing an empty particle set, so no CTD simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return - - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - - # deploy time for all ctds should be later than fieldset start time - if not all( - [np.datetime64(ctd.spacetime.time) >= fieldset_starttime for ctd in ctds] +# --------------- +# TODO: KERNELS +# --------------- + + +class CTDInputDataset(instruments.InputDataset): + """Input dataset for CTD instrument.""" + + def __init__(self): + """Initialise with instrument's name.""" + super().__init__(MYINSTRUMENT) + + def download_data(self, name: str) -> None: + """Download CTD data.""" + ... + + def get_dataset_path(self, name: str) -> Path: + """Get path to CTD dataset.""" + ... + + +class CTDInstrument(instruments.Instrument): + """CTD instrument class.""" + + def __init__( + self, + config, + input_dataset: CTDInputDataset, + kernels, ): - raise ValueError("CTD deployed before fieldset starts.") - - # depth the ctd will go to. shallowest between ctd max depth and bathymetry. - max_depths = [ - max( - ctd.max_depth, - fieldset.bathymetry.eval( - z=0, y=ctd.spacetime.location.lat, x=ctd.spacetime.location.lon, time=0 - ), - ) - for ctd in ctds - ] - - # CTD depth can not be too shallow, because kernel would break. - # This shallow is not useful anyway, no need to support. - if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): - raise ValueError( - f"CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" - ) - - # define parcel particles - ctd_particleset = ParticleSet( - fieldset=fieldset, - pclass=_CTDParticle, - lon=[ctd.spacetime.location.lon for ctd in ctds], - lat=[ctd.spacetime.location.lat for ctd in ctds], - depth=[ctd.min_depth for ctd in ctds], - time=[ctd.spacetime.time for ctd in ctds], - max_depth=max_depths, - min_depth=[ctd.min_depth for ctd in ctds], - winch_speed=[WINCH_SPEED for _ in ctds], - ) - - # define output file for the simulation - out_file = ctd_particleset.ParticleFile(name=out_path, outputdt=outputdt) - - # execute simulation - ctd_particleset.execute( - [_sample_salinity, _sample_temperature, _ctd_cast], - endtime=fieldset_endtime, - dt=DT, - verbose_progress=False, - output_file=out_file, - ) - - # there should be no particles left, as they delete themselves when they resurface - if len(ctd_particleset.particledata) != 0: - raise ValueError( - "Simulation ended before CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." - ) + """Initialise with instrument's name.""" + super().__init__(MYINSTRUMENT, config, input_dataset, kernels) + + def load_fieldset(self): + """Load fieldset.""" + ... + + def simulate(self): + """Simulate measurements.""" + ... diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py deleted file mode 100644 index 5aef240f..00000000 --- a/src/virtualship/instruments/drifter.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Drifter instrument.""" - -from dataclasses import dataclass -from datetime import datetime, timedelta -from pathlib import Path - -import numpy as np -from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable - -from virtualship.models import Spacetime - - -@dataclass -class Drifter: - """Configuration for a single Drifter.""" - - spacetime: Spacetime - depth: float # depth at which it floats and samples - lifetime: timedelta | None # if none, lifetime is infinite - - -_DrifterParticle = JITParticle.add_variables( - [ - Variable("temperature", dtype=np.float32, initial=np.nan), - Variable("has_lifetime", dtype=np.int8), # bool - Variable("age", dtype=np.float32, initial=0.0), - Variable("lifetime", dtype=np.float32), - ] -) - - -def _sample_temperature(particle, fieldset, time): - particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] - - -def _check_lifetime(particle, fieldset, time): - if particle.has_lifetime == 1: - particle.age += particle.dt - if particle.age >= particle.lifetime: - particle.delete() - - -def simulate_drifters( - fieldset: FieldSet, - out_path: str | Path, - drifters: list[Drifter], - outputdt: timedelta, - dt: timedelta, - endtime: datetime | None = None, -) -> None: - """ - Use Parcels to simulate a set of drifters in a fieldset. - - :param fieldset: The fieldset to simulate the Drifters in. - :param out_path: The path to write the results to. - :param drifters: A list of drifters to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation. - :param dt: Dt for integration. - :param endtime: Stop at this time, or if None, continue until the end of the fieldset or until all drifters ended. If this is earlier than the last drifter ended or later than the end of the fieldset, a warning will be printed. - """ - if len(drifters) == 0: - print( - "No drifters provided. Parcels currently crashes when providing an empty particle set, so no drifter simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return - - # define parcel particles - drifter_particleset = ParticleSet( - fieldset=fieldset, - pclass=_DrifterParticle, - lat=[drifter.spacetime.location.lat for drifter in drifters], - lon=[drifter.spacetime.location.lon for drifter in drifters], - depth=[drifter.depth for drifter in drifters], - time=[drifter.spacetime.time for drifter in drifters], - has_lifetime=[1 if drifter.lifetime is not None else 0 for drifter in drifters], - lifetime=[ - 0 if drifter.lifetime is None else drifter.lifetime.total_seconds() - for drifter in drifters - ], - ) - - # define output file for the simulation - out_file = drifter_particleset.ParticleFile( - name=out_path, outputdt=outputdt, chunks=[len(drifter_particleset), 100] - ) - - # get earliest between fieldset end time and provide end time - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - if endtime is None: - actual_endtime = fieldset_endtime - elif endtime > fieldset_endtime: - print("WARN: Requested end time later than fieldset end time.") - actual_endtime = fieldset_endtime - else: - actual_endtime = np.timedelta64(endtime) - - # execute simulation - drifter_particleset.execute( - [AdvectionRK4, _sample_temperature, _check_lifetime], - endtime=actual_endtime, - dt=dt, - output_file=out_file, - verbose_progress=True, - ) - - # if there are more particles left than the number of drifters with an indefinite endtime, warn the user - if len(drifter_particleset.particledata) > len( - [d for d in drifters if d.lifetime is None] - ): - print( - "WARN: Some drifters had a life time beyond the end time of the fieldset or the requested end time." - ) diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py deleted file mode 100644 index 7b08ad4b..00000000 --- a/src/virtualship/instruments/ship_underwater_st.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Ship salinity and temperature.""" - -from pathlib import Path - -import numpy as np -from parcels import FieldSet, ParticleSet, ScipyParticle, Variable - -from virtualship.models import Spacetime - -# we specifically use ScipyParticle because we have many small calls to execute -# there is some overhead with JITParticle and this ends up being significantly faster -_ShipSTParticle = ScipyParticle.add_variables( - [ - Variable("S", dtype=np.float32, initial=np.nan), - Variable("T", dtype=np.float32, initial=np.nan), - ] -) - - -# define function sampling Salinity -def _sample_salinity(particle, fieldset, time): - particle.S = fieldset.S[time, particle.depth, particle.lat, particle.lon] - - -# define function sampling Temperature -def _sample_temperature(particle, fieldset, time): - particle.T = fieldset.T[time, particle.depth, particle.lat, particle.lon] - - -def simulate_ship_underwater_st( - fieldset: FieldSet, - out_path: str | Path, - depth: float, - sample_points: list[Spacetime], -) -> None: - """ - Use Parcels to simulate underway data, measuring salinity and temperature at the given depth along the ship track in a fieldset. - - :param fieldset: The fieldset to simulate the sampling in. - :param out_path: The path to write the results to. - :param depth: The depth at which to measure. 0 is water surface, negative is into the water. - :param sample_points: The places and times to sample at. - """ - sample_points.sort(key=lambda p: p.time) - - particleset = ParticleSet.from_list( - fieldset=fieldset, - pclass=_ShipSTParticle, - lon=0.0, # initial lat/lon are irrelevant and will be overruled later - lat=0.0, - depth=depth, - time=0, # same for time - ) - - # define output file for the simulation - # outputdt set to infinie as we want to just want to write at the end of every call to 'execute' - out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) - - # iterate over each point, manually set lat lon time, then - # execute the particle set for one step, performing one set of measurement - for point in sample_points: - particleset.lon_nextloop[:] = point.location.lon - particleset.lat_nextloop[:] = point.location.lat - particleset.time_nextloop[:] = fieldset.time_origin.reltime( - np.datetime64(point.time) - ) - - # perform one step using the particleset - # dt and runtime are set so exactly one step is made. - particleset.execute( - [_sample_salinity, _sample_temperature], - dt=1, - runtime=1, - verbose_progress=False, - output_file=out_file, - ) diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py deleted file mode 100644 index 6d75be8c..00000000 --- a/src/virtualship/instruments/xbt.py +++ /dev/null @@ -1,141 +0,0 @@ -"""XBT instrument.""" - -from dataclasses import dataclass -from datetime import timedelta -from pathlib import Path - -import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable - -from virtualship.models import Spacetime - - -@dataclass -class XBT: - """Configuration for a single XBT.""" - - spacetime: Spacetime - min_depth: float - max_depth: float - fall_speed: float - deceleration_coefficient: float - - -_XBTParticle = JITParticle.add_variables( - [ - Variable("temperature", dtype=np.float32, initial=np.nan), - Variable("max_depth", dtype=np.float32), - Variable("min_depth", dtype=np.float32), - Variable("fall_speed", dtype=np.float32), - Variable("deceleration_coefficient", dtype=np.float32), - ] -) - - -def _sample_temperature(particle, fieldset, time): - particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] - - -def _xbt_cast(particle, fieldset, time): - particle_ddepth = -particle.fall_speed * particle.dt - - # update the fall speed from the quadractic fall-rate equation - # check https://doi.org/10.5194/os-7-231-2011 - particle.fall_speed = ( - particle.fall_speed - 2 * particle.deceleration_coefficient * particle.dt - ) - - # delete particle if depth is exactly max_depth - if particle.depth == particle.max_depth: - particle.delete() - - # set particle depth to max depth if it's too deep - if particle.depth + particle_ddepth < particle.max_depth: - particle_ddepth = particle.max_depth - particle.depth - - -def simulate_xbt( - fieldset: FieldSet, - out_path: str | Path, - xbts: list[XBT], - outputdt: timedelta, -) -> None: - """ - Use Parcels to simulate a set of XBTs in a fieldset. - - :param fieldset: The fieldset to simulate the XBTs in. - :param out_path: The path to write the results to. - :param xbts: A list of XBTs to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :raises ValueError: Whenever provided XBTs, fieldset, are not compatible with this function. - """ - DT = 10.0 # dt of XBT simulation integrator - - if len(xbts) == 0: - print( - "No XBTs provided. Parcels currently crashes when providing an empty particle set, so no XBT simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return - - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - - # deploy time for all xbts should be later than fieldset start time - if not all( - [np.datetime64(xbt.spacetime.time) >= fieldset_starttime for xbt in xbts] - ): - raise ValueError("XBT deployed before fieldset starts.") - - # depth the xbt will go to. shallowest between xbt max depth and bathymetry. - max_depths = [ - max( - xbt.max_depth, - fieldset.bathymetry.eval( - z=0, y=xbt.spacetime.location.lat, x=xbt.spacetime.location.lon, time=0 - ), - ) - for xbt in xbts - ] - - # initial fall speeds - initial_fall_speeds = [xbt.fall_speed for xbt in xbts] - - # XBT depth can not be too shallow, because kernel would break. - # This shallow is not useful anyway, no need to support. - for max_depth, fall_speed in zip(max_depths, initial_fall_speeds, strict=False): - if not max_depth <= -DT * fall_speed: - raise ValueError( - f"XBT max_depth or bathymetry shallower than maximum {-DT * fall_speed}" - ) - - # define xbt particles - xbt_particleset = ParticleSet( - fieldset=fieldset, - pclass=_XBTParticle, - lon=[xbt.spacetime.location.lon for xbt in xbts], - lat=[xbt.spacetime.location.lat for xbt in xbts], - depth=[xbt.min_depth for xbt in xbts], - time=[xbt.spacetime.time for xbt in xbts], - max_depth=max_depths, - min_depth=[xbt.min_depth for xbt in xbts], - fall_speed=[xbt.fall_speed for xbt in xbts], - ) - - # define output file for the simulation - out_file = xbt_particleset.ParticleFile(name=out_path, outputdt=outputdt) - - # execute simulation - xbt_particleset.execute( - [_sample_temperature, _xbt_cast], - endtime=fieldset_endtime, - dt=DT, - verbose_progress=False, - output_file=out_file, - ) - - # there should be no particles left, as they delete themselves when they finish profiling - if len(xbt_particleset.particledata) != 0: - raise ValueError( - "Simulation ended before XBT finished profiling. This most likely means the field time dimension did not match the simulation time span." - ) diff --git a/src/virtualship/models/instruments.py b/src/virtualship/models/instruments.py new file mode 100644 index 00000000..8b5e701e --- /dev/null +++ b/src/virtualship/models/instruments.py @@ -0,0 +1,82 @@ +import abc +from collections.abc import Callable +from pathlib import Path + +from yaspin import yaspin + +from virtualship.utils import ( + ship_spinner, +) + +# TODO +# how much detail needs to be fed into InputDataset (i.e. how much it differs per instrument) +# may impact whether need a child class (e.g. CTDInputDataset) as well as just InputDataset +# or whether it could just be fed a `name` ... ? + +# ++ abc.abstractmethods could be useful for testing purposes...e.g. will fail if an instrumnet implementation doesn't adhere to the `Instrument` class standards + + +class InputDataset(abc.ABC): + """Base class for instrument input datasets.""" + + def __init__(self, name): + """Initialise input dataset.""" + self.name = name + + @abc.abstractmethod + def download_data(self, name: str) -> None: + """Download data for the instrument.""" + pass + + @abc.abstractmethod + def get_dataset_path(self, name: str) -> Path: + """Get path to the dataset.""" + pass + + +class Instrument(abc.ABC): + """Base class for instruments.""" + + def __init__( + self, + name: str, + config, + input_dataset: InputDataset, + kernels: list[Callable], + ): + """Initialise instrument.""" + self.name = name + self.config = config + self.input_dataset = input_dataset + self.kernels = kernels + + @abc.abstractmethod + def load_fieldset(self): + """Load fieldset for simulation.""" + pass + + def get_output_path(self, output_dir: Path) -> Path: + """Get output path for results.""" + return output_dir / f"{self.name}.zarr" + + def run(self): + """Run instrument simulation.""" + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate() + spinner.ok("✅") + + @abc.abstractmethod + def simulate(self): + """Simulate instrument measurements.""" + pass + + +# e.g. pseudo-code ... +# TODO: (necessary?) how to dynamically assemble list of all instruments defined so that new instruments can be added only by changes in one place...? +available_instruments: list = ... +# for instrument in available_instruments: +# MyInstrument(instrument) From b4fbf2be6090173c8127bd668a6482b41978d496 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 18 Aug 2025 21:07:41 +0100 Subject: [PATCH 04/97] ignore refactoring notes in gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 4efdfe45..bd70d1d5 100644 --- a/.gitignore +++ b/.gitignore @@ -178,3 +178,7 @@ src/virtualship/_version_setup.py .vscode/ .DS_Store + + +# Ignore temporary notes files for refactoring +_refactoring_notes/ From 2409517f0dd3302a3b8ef40b1514e6e497672cc4 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 18 Aug 2025 21:08:44 +0100 Subject: [PATCH 05/97] add note to remove upon completing v1 dev --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index bd70d1d5..d9903e71 100644 --- a/.gitignore +++ b/.gitignore @@ -181,4 +181,5 @@ src/virtualship/_version_setup.py # Ignore temporary notes files for refactoring +# TODO: remove when finished with v1 dev! _refactoring_notes/ From cfa7a0f840861ac96c65f31c3b7d41c13c994f79 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 17 Sep 2025 14:58:59 +0100 Subject: [PATCH 06/97] scratch inputdataset objects integration to _fetch --- .gitignore | 5 - src/virtualship/cli/_fetch.py | 184 ++++----------- src/virtualship/instruments/ctd.py | 59 +++-- src/virtualship/instruments/ctd_bgc.py | 80 +++++++ src/virtualship/instruments/master.py | 66 ++++++ src/virtualship/models/instruments.py | 87 +++++-- src/virtualship/models/schedule.py | 236 +++++++++++++++++++ src/virtualship/models/ship_config.py | 310 +++++++++++++++++++++++++ 8 files changed, 850 insertions(+), 177 deletions(-) create mode 100644 src/virtualship/instruments/master.py create mode 100644 src/virtualship/models/schedule.py create mode 100644 src/virtualship/models/ship_config.py diff --git a/.gitignore b/.gitignore index d9903e71..4efdfe45 100644 --- a/.gitignore +++ b/.gitignore @@ -178,8 +178,3 @@ src/virtualship/_version_setup.py .vscode/ .DS_Store - - -# Ignore temporary notes files for refactoring -# TODO: remove when finished with v1 dev! -_refactoring_notes/ diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index bf35fe2f..127ee256 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -2,10 +2,12 @@ import hashlib import shutil -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING +import copernicusmarine +from copernicusmarine.core_functions.credentials_utils import InvalidUsernameOrPassword from pydantic import BaseModel from virtualship.errors import IncompleteDownloadError @@ -19,11 +21,10 @@ from virtualship.models import SpaceTimeRegion import click -import copernicusmarine -from copernicusmarine.core_functions.credentials_utils import InvalidUsernameOrPassword import virtualship.cli._creds as creds from virtualship.utils import EXPEDITION +from virtualship.instruments.master import INSTRUMENTS DOWNLOAD_METADATA = "download_metadata.yaml" @@ -38,15 +39,13 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None be provided on prompt, via command line arguments, or via a YAML config file. Run `virtualship fetch` on an expedition for more info. """ - from virtualship.models import InstrumentType - if sum([username is None, password is None]) == 1: raise ValueError("Both username and password must be provided when using CLI.") path = Path(path) - data_folder = path / "data" - data_folder.mkdir(exist_ok=True) + data_dir = path / "data" + data_dir.mkdir(exist_ok=True) expedition = _get_expedition(path) @@ -61,7 +60,8 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None expedition.schedule.space_time_region ) - existing_download = get_existing_download(data_folder, space_time_region_hash) + # TODO: this (below) probably needs updating! + existing_download = get_existing_download(data_dir, space_time_region_hash) if existing_download is not None: click.echo( f"Data download for space-time region already completed ('{existing_download}')." @@ -69,7 +69,10 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None return creds_path = path / creds.CREDENTIALS_FILE - username, password = creds.get_credentials_flow(username, password, creds_path) + credentials = {} + credentials["username"], credentials["password"] = creds.get_credentials_flow( + username, password, creds_path + ) # Extract space_time_region details from the schedule spatial_range = expedition.schedule.space_time_region.spatial_range @@ -79,13 +82,46 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None instruments_in_schedule = expedition.schedule.get_instruments() # Create download folder and set download metadata - download_folder = data_folder / hash_to_filename(space_time_region_hash) + download_folder = data_dir / hash_to_filename(space_time_region_hash) download_folder.mkdir() DownloadMetadata(download_complete=False).to_yaml( download_folder / DOWNLOAD_METADATA ) shutil.copyfile(path / EXPEDITION, download_folder / EXPEDITION) + # bathymetry (required for all expeditions) + copernicusmarine.subset( + dataset_id="cmems_mod_glo_phy_my_0.083deg_static", + variables=["deptho"], + minimum_longitude=space_time_region.spatial_range.minimum_longitude, + maximum_longitude=space_time_region.spatial_range.maximum_longitude, + minimum_latitude=space_time_region.spatial_range.minimum_latitude, + maximum_latitude=space_time_region.spatial_range.maximum_latitude, + start_datetime=space_time_region.time_range.start_time, + end_datetime=space_time_region.time_range.start_time, + minimum_depth=abs(space_time_region.spatial_range.minimum_depth), + maximum_depth=abs(space_time_region.spatial_range.maximum_depth), + output_filename="bathymetry.nc", + output_directory=download_folder, + username=credentials["username"], + password=credentials["password"], + overwrite=True, + coordinates_selection_method="outside", + ) + + # keep only instruments in INTSTRUMENTS which are in schedule + filtered_instruments = { + k: v for k, v in INSTRUMENTS.items() if k in instruments_in_schedule + } + + # iterate across instruments and download data based on space_time_region + for _, instrument in filtered_instruments.items(): + try: + instrument["input_class"]( + data_dir=download_folder, + credentials=credentials, + space_time_region=space_time_region, + ) #! #### TODO # ++ new logic here where iterates (?) through available instruments and determines whether download is required: @@ -198,127 +234,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None shutil.rmtree(download_folder) raise e - click.echo("Drifter data download based on space-time region completed.") - - if InstrumentType.ARGO_FLOAT in instruments_in_schedule: - print("Argo float data will be downloaded. Please wait...") - argo_download_dict = { - "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", - "variables": ["uo", "vo"], - "output_filename": "argo_float_uv.nc", - }, - "Sdata": { - "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", - "variables": ["so"], - "output_filename": "argo_float_s.nc", - }, - "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", - "variables": ["thetao"], - "output_filename": "argo_float_t.nc", - }, - } - - # Iterate over all datasets and download each based on space_time_region - try: - for dataset in argo_download_dict.values(): - copernicusmarine.subset( - dataset_id=dataset["dataset_id"], - variables=dataset["variables"], - minimum_longitude=spatial_range.minimum_longitude - 3.0, - maximum_longitude=spatial_range.maximum_longitude + 3.0, - minimum_latitude=spatial_range.minimum_latitude - 3.0, - maximum_latitude=spatial_range.maximum_latitude + 3.0, - start_datetime=start_datetime, - end_datetime=end_datetime + timedelta(days=21), - minimum_depth=abs(1), - maximum_depth=abs(spatial_range.maximum_depth), - output_filename=dataset["output_filename"], - output_directory=download_folder, - username=username, - password=password, - overwrite=True, - coordinates_selection_method="outside", - ) - except InvalidUsernameOrPassword as e: - shutil.rmtree(download_folder) - raise e - - click.echo("Argo_float data download based on space-time region completed.") - - if InstrumentType.CTD_BGC in instruments_in_schedule: - print("CTD_BGC data will be downloaded. Please wait...") - - ctd_bgc_download_dict = { - "o2data": { - "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", - "variables": ["o2"], - "output_filename": "ctd_bgc_o2.nc", - }, - "chlorodata": { - "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", - "variables": ["chl"], - "output_filename": "ctd_bgc_chl.nc", - }, - "nitratedata": { - "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", - "variables": ["no3"], - "output_filename": "ctd_bgc_no3.nc", - }, - "phosphatedata": { - "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", - "variables": ["po4"], - "output_filename": "ctd_bgc_po4.nc", - }, - "phdata": { - "dataset_id": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", - "variables": ["ph"], - "output_filename": "ctd_bgc_ph.nc", - }, - "phytoplanktondata": { - "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", - "variables": ["phyc"], - "output_filename": "ctd_bgc_phyc.nc", - }, - "zooplanktondata": { - "dataset_id": "cmems_mod_glo_bgc-plankton_anfc_0.25deg_P1D-m", - "variables": ["zooc"], - "output_filename": "ctd_bgc_zooc.nc", - }, - "primaryproductiondata": { - "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", - "variables": ["nppv"], - "output_filename": "ctd_bgc_nppv.nc", - }, - } - - # Iterate over all datasets and download each based on space_time_region - try: - for dataset in ctd_bgc_download_dict.values(): - copernicusmarine.subset( - dataset_id=dataset["dataset_id"], - variables=dataset["variables"], - minimum_longitude=spatial_range.minimum_longitude - 3.0, - maximum_longitude=spatial_range.maximum_longitude + 3.0, - minimum_latitude=spatial_range.minimum_latitude - 3.0, - maximum_latitude=spatial_range.maximum_latitude + 3.0, - start_datetime=start_datetime, - end_datetime=end_datetime + timedelta(days=21), - minimum_depth=abs(1), - maximum_depth=abs(spatial_range.maximum_depth), - output_filename=dataset["output_filename"], - output_directory=download_folder, - username=username, - password=password, - overwrite=True, - coordinates_selection_method="outside", - ) - except InvalidUsernameOrPassword as e: - shutil.rmtree(download_folder) - raise e - - click.echo("CTD_BGC data download based on space-time region completed.") + click.echo(f"{instrument.name} data download completed.") # TODO complete_download(download_folder) @@ -386,11 +302,9 @@ def from_yaml(cls, file_path: str | Path) -> DownloadMetadata: return _generic_load_yaml(file_path, cls) -def get_existing_download( - data_folder: Path, space_time_region_hash: str -) -> Path | None: +def get_existing_download(data_dir: Path, space_time_region_hash: str) -> Path | None: """Check if a download has already been completed. If so, return the path for existing download.""" - for download_path in data_folder.rglob("*"): + for download_path in data_dir.rglob("*"): try: hash = filename_to_hash(download_path.name) except ValueError: diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 0d7294fb..624256cb 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -1,15 +1,17 @@ from dataclasses import dataclass -from pathlib import Path +from typing import ClassVar from virtualship.models import Spacetime, instruments -MYINSTRUMENT = "CTD" +## TODO: __init__.py will also need updating! +# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py @dataclass class CTD: """CTD configuration.""" + name: ClassVar[str] = "CTD" spacetime: Spacetime min_depth: float max_depth: float @@ -23,17 +25,36 @@ class CTD: class CTDInputDataset(instruments.InputDataset): """Input dataset for CTD instrument.""" - def __init__(self): - """Initialise with instrument's name.""" - super().__init__(MYINSTRUMENT) - - def download_data(self, name: str) -> None: - """Download CTD data.""" - ... + DOWNLOAD_BUFFERS: ClassVar[dict] = { + "latlon_degrees": 0.0, + "days": 0.0, + } # CTD data requires no buffers - def get_dataset_path(self, name: str) -> Path: - """Get path to CTD dataset.""" - ... + def __init__(self, data_dir, credentials, space_time_region): + """Initialise with instrument's name.""" + super().__init__( + CTD.name, + self.DOWNLOAD_BUFFERS["latlon_degrees"], + self.DOWNLOAD_BUFFERS["days"], + data_dir, + credentials, + space_time_region, + ) + + def get_datasets_dict(self) -> dict: + """Get variable specific args for instrument.""" + return { + "Sdata": { + "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "variables": ["so"], + "output_filename": f"{self.name}_s.nc", + }, + "Tdata": { + "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "variables": ["thetao"], + "output_filename": f"{self.name}_t.nc", + }, + } class CTDInstrument(instruments.Instrument): @@ -42,16 +63,18 @@ class CTDInstrument(instruments.Instrument): def __init__( self, config, - input_dataset: CTDInputDataset, + input_dataset, kernels, ): """Initialise with instrument's name.""" - super().__init__(MYINSTRUMENT, config, input_dataset, kernels) - - def load_fieldset(self): - """Load fieldset.""" - ... + super().__init__(CTD.name, config, input_dataset, kernels) def simulate(self): """Simulate measurements.""" ... + + +# # [PSEUDO-CODE] example implementation for reference +# ctd = CTDInstrument(config=CTD, data_dir=..., kernels=...) + +# ctd.simulate(...) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index fde92ca1..6b9b2f29 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -1,3 +1,83 @@ +# from dataclasses import dataclass +# from typing import ClassVar + +# from virtualship.models import Spacetime, instruments + +# MYINSTRUMENT = "CTD_BGC" + + +# @dataclass +# class CTD_BGC: +# """CTD_BGC configuration.""" + +# spacetime: Spacetime +# min_depth: float +# max_depth: float + + +# # --------------- +# # TODO: KERNELS +# # --------------- + + +# class CTD_BGCInputDataset(instruments.InputDataset): +# """Input dataset object for CTD_BGC instrument.""" + +# DOWNLOAD_BUFFERS: ClassVar[dict] = { +# "latlon_degrees": 0.0, +# "days": 0.0, +# } # CTD_BGC data requires no buffers + +# def __init__(self, data_dir, credentials, space_time_region): +# """Initialise with instrument's name.""" +# super().__init__( +# MYINSTRUMENT, +# self.DOWNLOAD_BUFFERS["latlon_degrees"], +# self.DOWNLOAD_BUFFERS["days"], +# data_dir, +# credentials, +# space_time_region, +# ) + +# def datasets_dir(self) -> dict: +# """Variable specific args for instrument.""" +# return { +# "o2data": { +# "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", +# "variables": ["o2"], +# "output_filename": "ctd_bgc_o2.nc", +# }, +# "chlorodata": { +# "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", +# "variables": ["chl"], +# "output_filename": "ctd_bgc_chloro.nc", +# }, +# } + + +# class CTD_BGCInstrument(instruments.Instrument): +# """CTD_BGC instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(MYINSTRUMENT, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +# # # [PSEUDO-CODE] example implementation for reference +# # ctd = CTD_BGCInstrument(config=CTD_BGC, data_dir=..., kernels=...) + +# # ctd.simulate(...) + + """CTD_BGC instrument.""" from dataclasses import dataclass diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py new file mode 100644 index 00000000..4705c8e3 --- /dev/null +++ b/src/virtualship/instruments/master.py @@ -0,0 +1,66 @@ +# + +# TODO: temporary measure so as not to have to overhaul the InstrumentType class logic in one go +#! And also to avoid breaking other parts of the codebase which rely on InstrumentType when for now just working on fetch +# TODO: ideally this can evaporate... +# TODO: discuss to see if there's a better option...! + +from enum import Enum + +# and so on ... +# from virtualship.instruments.ctd import CTDInputDataset, CTDInstrument + + +class InstrumentType(Enum): + """Types of the instruments.""" + + CTD = "CTD" + # CTD_BGC = "CTD_BGC" + # DRIFTER = "DRIFTER" + # ARGO_FLOAT = "ARGO_FLOAT" + # XBT = "XBT" + + # # TODO: should underway also be handled here?! + # ADCP = "ADCP" + # UNDERWAY_ST = "UNDERWAY_ST" + + +# replace with imports instead... +class CTDInputDataset: + """Input dataset class for CTD instrument.""" + + pass + + +class CTDInstrument: + """Instrument class for CTD instrument.""" + + pass + + +INSTRUMENTS = { + inst: { + "input_class": globals()[f"{inst.value}InputDataset"], + "instrument_class": globals()[f"{inst.value}Instrument"], + } + for inst in InstrumentType + if f"{inst.value}InputDataset" in globals() + and f"{inst.value}Instrument" in globals() +} + + +# INSTRUMENTS = { +# InstrumentType.CTD: { +# "input_class": CTDInputDataset, +# "instrument_class": CTDInstrument, +# } +# # and so on for other instruments... +# } + +# INSTRUMENTS = { +# "InstrumentType.CTD": { +# "input_class": "test", +# "instrument_class": "test", +# } +# # and so on for other instruments... +# } diff --git a/src/virtualship/models/instruments.py b/src/virtualship/models/instruments.py index 8b5e701e..cddebd85 100644 --- a/src/virtualship/models/instruments.py +++ b/src/virtualship/models/instruments.py @@ -1,37 +1,86 @@ import abc from collections.abc import Callable +from datetime import timedelta from pathlib import Path +import copernicusmarine from yaspin import yaspin -from virtualship.utils import ( - ship_spinner, -) +from virtualship.models.space_time_region import SpaceTimeRegion +from virtualship.utils import ship_spinner -# TODO +# TODO list START # how much detail needs to be fed into InputDataset (i.e. how much it differs per instrument) # may impact whether need a child class (e.g. CTDInputDataset) as well as just InputDataset # or whether it could just be fed a `name` ... ? # ++ abc.abstractmethods could be useful for testing purposes...e.g. will fail if an instrumnet implementation doesn't adhere to the `Instrument` class standards +# ++ discussion point with others, do we think it's okay to overhaul the data downloading so that each instrument has it's own files, rather than sharing data? +# ++ it's a cleaner way of making the whole repo more modular, i.e. have higher order logic for defining data downloads and housing all instrument logic in one place... +# ++ may even not matter so much considering working towards cloud integration... + we are not looking to optimise performance...? +# ++ OR, for now work on it in this way and then at the end make some clever changes to consolidate to minimum number of files dependent on instrument selections...? +# TODO list END + class InputDataset(abc.ABC): """Base class for instrument input datasets.""" - def __init__(self, name): + def __init__( + self, + name: str, + latlon_buffer: float, + datetime_buffer: float, + data_dir: str, + credentials: dict, + space_time_region: SpaceTimeRegion, + ): """Initialise input dataset.""" self.name = name + self.latlon_buffer = latlon_buffer + self.datetime_buffer = datetime_buffer + self.data_dir = data_dir + self.credentials = credentials + self.space_time_region = space_time_region @abc.abstractmethod - def download_data(self, name: str) -> None: - """Download data for the instrument.""" - pass - - @abc.abstractmethod - def get_dataset_path(self, name: str) -> Path: - """Get path to the dataset.""" - pass + def get_datasets_dict(self) -> dict: + """Get parameters for instrument's variable(s) specific data download.""" + ... + + def download_data(self) -> None: + """Download data for the instrument using copernicusmarine.""" + parameter_args = dict( + minimum_longitude=self.space_time_region.spatial_range.minimum_longitude + - self.latlon_buffer, + maximum_longitude=self.space_time_region.spatial_range.maximum_longitude + + self.latlon_buffer, + minimum_latitude=self.space_time_region.spatial_range.minimum_latitude + - self.latlon_buffer, + maximum_latitude=self.space_time_region.spatial_range.maximum_latitude + + self.latlon_buffer, + start_datetime=self.space_time_region.time_range.start_time, + end_datetime=self.space_time_region.time_range.end_time + + timedelta(days=self.datetime_buffer), + minimum_depth=abs(self.space_time_region.spatial_range.minimum_depth), + maximum_depth=abs(self.space_time_region.spatial_range.maximum_depth), + output_directory=self.data_dir, + username=self.credentials["username"], + password=self.credentials["password"], + overwrite=True, + coordinates_selection_method="outside", + ) + + datasets_args = self.get_datasets_dict() + + for dataset in datasets_args.values(): + download_args = {**parameter_args, **dataset} + copernicusmarine.subset(**download_args) + + # def get_fieldset_paths(self) -> list: + # """List of paths for instrument's (downloaded) input data.""" + + # ... class Instrument(abc.ABC): @@ -47,13 +96,13 @@ def __init__( """Initialise instrument.""" self.name = name self.config = config - self.input_dataset = input_dataset + self.input_data = input_dataset self.kernels = kernels - @abc.abstractmethod - def load_fieldset(self): - """Load fieldset for simulation.""" - pass + # def load_fieldset(self): + # """Load fieldset for simulation.""" + # # paths = self.input_data.get_fieldset_paths() + # ... def get_output_path(self, output_dir: Path) -> Path: """Get output path for results.""" @@ -72,7 +121,7 @@ def run(self): @abc.abstractmethod def simulate(self): """Simulate instrument measurements.""" - pass + ... # e.g. pseudo-code ... diff --git a/src/virtualship/models/schedule.py b/src/virtualship/models/schedule.py new file mode 100644 index 00000000..091c23b4 --- /dev/null +++ b/src/virtualship/models/schedule.py @@ -0,0 +1,236 @@ +"""Schedule class.""" + +from __future__ import annotations + +import itertools +from datetime import datetime, timedelta +from pathlib import Path +from typing import TYPE_CHECKING + +import pydantic +import pyproj +import yaml + +from virtualship.errors import ScheduleError + +from .instruments import InstrumentType +from .location import Location +from .space_time_region import SpaceTimeRegion + +if TYPE_CHECKING: + from parcels import FieldSet + + from virtualship.expedition.input_data import InputData + +projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") + + +class Waypoint(pydantic.BaseModel): + """A Waypoint to sail to with an optional time and an optional instrument.""" + + location: Location + time: datetime | None = None + instrument: InstrumentType | list[InstrumentType] | None = None + + @pydantic.field_serializer("instrument") + def serialize_instrument(self, instrument): + """Ensure InstrumentType is serialized as a string (or list of strings).""" + if isinstance(instrument, list): + return [inst.value for inst in instrument] + return instrument.value if instrument else None + + +class Schedule(pydantic.BaseModel): + """Schedule of the virtual ship.""" + + waypoints: list[Waypoint] + space_time_region: SpaceTimeRegion | None = None + + model_config = pydantic.ConfigDict(extra="forbid") + + def to_yaml(self, file_path: str | Path) -> None: + """ + Write schedule to yaml file. + + :param file_path: Path to the file to write to. + """ + with open(file_path, "w") as file: + yaml.dump( + self.model_dump( + by_alias=True, + ), + file, + ) + + @classmethod + def from_yaml(cls, file_path: str | Path) -> Schedule: + """ + Load schedule from yaml file. + + :param file_path: Path to the file to load from. + :returns: The schedule. + """ + with open(file_path) as file: + data = yaml.safe_load(file) + return Schedule(**data) + + def get_instruments(self) -> set[InstrumentType]: + """ + Retrieve a set of unique instruments used in the schedule. + + This method iterates through all waypoints in the schedule and collects + the instruments associated with each waypoint. It returns a set of unique + instruments, either as objects or as names. + + :raises CheckpointError: If the past waypoints in the given schedule + have been changed compared to the checkpoint. + :return: set: A set of unique instruments used in the schedule. + + """ + instruments_in_schedule = [] + for waypoint in self.waypoints: + if waypoint.instrument: + for instrument in waypoint.instrument: + if instrument: + instruments_in_schedule.append(instrument) + return set(instruments_in_schedule) + + def verify( + self, + ship_speed: float, + input_data: InputData | None, + *, + check_space_time_region: bool = False, + ignore_missing_fieldsets: bool = False, + ) -> None: + """ + Verify the feasibility and correctness of the schedule's waypoints. + + This method checks various conditions to ensure the schedule is valid: + 1. At least one waypoint is provided. + 2. The first waypoint has a specified time. + 3. Waypoint times are in ascending order. + 4. All waypoints are in water (not on land). + 5. The ship can arrive on time at each waypoint given its speed. + + :param ship_speed: The ship's speed in knots. + :param input_data: An InputData object containing fieldsets used to check if waypoints are on water. + :param check_space_time_region: whether to check for missing space_time_region. + :param ignore_missing_fieldsets: whether to ignore warning for missing field sets. + :raises PlanningError: If any of the verification checks fail, indicating infeasible or incorrect waypoints. + :raises NotImplementedError: If an instrument in the schedule is not implemented. + :return: None. The method doesn't return a value but raises exceptions if verification fails. + """ + print("\nVerifying route... ") + + if check_space_time_region and self.space_time_region is None: + raise ScheduleError( + "space_time_region not found in schedule, please define it to fetch the data." + ) + + if len(self.waypoints) == 0: + raise ScheduleError("At least one waypoint must be provided.") + + # check first waypoint has a time + if self.waypoints[0].time is None: + raise ScheduleError("First waypoint must have a specified time.") + + # check waypoint times are in ascending order + timed_waypoints = [wp for wp in self.waypoints if wp.time is not None] + checks = [ + next.time >= cur.time for cur, next in itertools.pairwise(timed_waypoints) + ] + if not all(checks): + invalid_i = [i for i, c in enumerate(checks) if c] + raise ScheduleError( + f"Waypoint(s) {', '.join(f'#{i + 1}' for i in invalid_i)}: each waypoint should be timed after all previous waypoints", + ) + + # check if all waypoints are in water + # this is done by picking an arbitrary provided fieldset and checking if UV is not zero + + # get all available fieldsets + available_fieldsets = [] + if input_data is not None: + fieldsets = [ + input_data.adcp_fieldset, + input_data.argo_float_fieldset, + input_data.ctd_fieldset, + input_data.drifter_fieldset, + input_data.ship_underwater_st_fieldset, + ] + for fs in fieldsets: + if fs is not None: + available_fieldsets.append(fs) + + # check if there are any fieldsets, else it's an error + if len(available_fieldsets) == 0: + if not ignore_missing_fieldsets: + print( + "Cannot verify because no fieldsets have been loaded. This is probably " + "because you are not using any instruments in your schedule. This is not a problem, " + "but carefully check your waypoint locations manually." + ) + + else: + # pick any + fieldset = available_fieldsets[0] + # get waypoints with 0 UV + land_waypoints = [ + (wp_i, wp) + for wp_i, wp in enumerate(self.waypoints) + if _is_on_land_zero_uv(fieldset, wp) + ] + # raise an error if there are any + if len(land_waypoints) > 0: + raise ScheduleError( + f"The following waypoints are on land: {['#' + str(wp_i) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" + ) + + # check that ship will arrive on time at each waypoint (in case no unexpected event happen) + time = self.waypoints[0].time + for wp_i, (wp, wp_next) in enumerate( + zip(self.waypoints, self.waypoints[1:], strict=False) + ): + if wp.instrument is InstrumentType.CTD: + time += timedelta(minutes=20) + + geodinv: tuple[float, float, float] = projection.inv( + wp.location.lon, + wp.location.lat, + wp_next.location.lon, + wp_next.location.lat, + ) + distance = geodinv[2] + + time_to_reach = timedelta(seconds=distance / ship_speed * 3600 / 1852) + arrival_time = time + time_to_reach + + if wp_next.time is None: + time = arrival_time + elif arrival_time > wp_next.time: + raise ScheduleError( + f"Waypoint planning is not valid: would arrive too late at waypoint number {wp_i + 2}. " + f"location: {wp_next.location} time: {wp_next.time} instrument: {wp_next.instrument}" + ) + else: + time = wp_next.time + + print("... All good to go!") + + +def _is_on_land_zero_uv(fieldset: FieldSet, waypoint: Waypoint) -> bool: + """ + Check if waypoint is on land by assuming zero velocity means land. + + :param fieldset: The fieldset to sample the velocity from. + :param waypoint: The waypoint to check. + :returns: If the waypoint is on land. + """ + return fieldset.UV.eval( + 0, + fieldset.gridset.grids[0].depth[0], + waypoint.location.lat, + waypoint.location.lon, + applyConversion=False, + ) == (0.0, 0.0) diff --git a/src/virtualship/models/ship_config.py b/src/virtualship/models/ship_config.py new file mode 100644 index 00000000..61d3d390 --- /dev/null +++ b/src/virtualship/models/ship_config.py @@ -0,0 +1,310 @@ +"""ShipConfig and supporting classes.""" + +from __future__ import annotations + +from datetime import timedelta +from pathlib import Path +from typing import TYPE_CHECKING + +import pydantic +import yaml + +from virtualship.errors import ConfigError +from virtualship.models.instruments import InstrumentType +from virtualship.utils import _validate_numeric_mins_to_timedelta + +if TYPE_CHECKING: + from .schedule import Schedule + + +class ArgoFloatConfig(pydantic.BaseModel): + """Configuration for argos floats.""" + + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + drift_depth_meter: float = pydantic.Field(le=0.0) + vertical_speed_meter_per_second: float = pydantic.Field(lt=0.0) + cycle_days: float = pydantic.Field(gt=0.0) + drift_days: float = pydantic.Field(gt=0.0) + + +class ADCPConfig(pydantic.BaseModel): + """Configuration for ADCP instrument.""" + + max_depth_meter: float = pydantic.Field(le=0.0) + num_bins: int = pydantic.Field(gt=0.0) + period: timedelta = pydantic.Field( + serialization_alias="period_minutes", + validation_alias="period_minutes", + gt=timedelta(), + ) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("period") + def _serialize_period(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("period", mode="before") + def _validate_period(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class CTDConfig(pydantic.BaseModel): + """Configuration for CTD instrument.""" + + stationkeeping_time: timedelta = pydantic.Field( + serialization_alias="stationkeeping_time_minutes", + validation_alias="stationkeeping_time_minutes", + gt=timedelta(), + ) + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("stationkeeping_time") + def _serialize_stationkeeping_time(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("stationkeeping_time", mode="before") + def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class CTD_BGCConfig(pydantic.BaseModel): + """Configuration for CTD_BGC instrument.""" + + stationkeeping_time: timedelta = pydantic.Field( + serialization_alias="stationkeeping_time_minutes", + validation_alias="stationkeeping_time_minutes", + gt=timedelta(), + ) + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("stationkeeping_time") + def _serialize_stationkeeping_time(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("stationkeeping_time", mode="before") + def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class ShipUnderwaterSTConfig(pydantic.BaseModel): + """Configuration for underwater ST.""" + + period: timedelta = pydantic.Field( + serialization_alias="period_minutes", + validation_alias="period_minutes", + gt=timedelta(), + ) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("period") + def _serialize_period(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("period", mode="before") + def _validate_period(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class DrifterConfig(pydantic.BaseModel): + """Configuration for drifters.""" + + depth_meter: float = pydantic.Field(le=0.0) + lifetime: timedelta = pydantic.Field( + serialization_alias="lifetime_minutes", + validation_alias="lifetime_minutes", + gt=timedelta(), + ) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("lifetime") + def _serialize_lifetime(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("lifetime", mode="before") + def _validate_lifetime(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class XBTConfig(pydantic.BaseModel): + """Configuration for xbt instrument.""" + + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + fall_speed_meter_per_second: float = pydantic.Field(gt=0.0) + deceleration_coefficient: float = pydantic.Field(gt=0.0) + + +class ShipConfig(pydantic.BaseModel): + """Configuration of the virtual ship.""" + + ship_speed_knots: float = pydantic.Field(gt=0.0) + """ + Velocity of the ship in knots. + """ + + argo_float_config: ArgoFloatConfig | None = None + """ + Argo float configuration. + + If None, no argo floats can be deployed. + """ + + adcp_config: ADCPConfig | None = None + """ + ADCP configuration. + + If None, no ADCP measurements will be performed. + """ + + ctd_config: CTDConfig | None = None + """ + CTD configuration. + + If None, no CTDs can be cast. + """ + + ctd_bgc_config: CTD_BGCConfig | None = None + """ + CTD_BGC configuration. + + If None, no BGC CTDs can be cast. + """ + + ship_underwater_st_config: ShipUnderwaterSTConfig | None = None + """ + Ship underwater salinity temperature measurementconfiguration. + + If None, no ST measurements will be performed. + """ + + drifter_config: DrifterConfig | None = None + """ + Drifter configuration. + + If None, no drifters can be deployed. + """ + + xbt_config: XBTConfig | None = None + """ + XBT configuration. + + If None, no XBTs can be cast. + """ + + model_config = pydantic.ConfigDict(extra="forbid") + + def to_yaml(self, file_path: str | Path) -> None: + """ + Write config to yaml file. + + :param file_path: Path to the file to write to. + """ + with open(file_path, "w") as file: + yaml.dump(self.model_dump(by_alias=True), file) + + @classmethod + def from_yaml(cls, file_path: str | Path) -> ShipConfig: + """ + Load config from yaml file. + + :param file_path: Path to the file to load from. + :returns: The config. + """ + with open(file_path) as file: + data = yaml.safe_load(file) + return ShipConfig(**data) + + def verify(self, schedule: Schedule) -> None: + """ + Verify the ship configuration against the provided schedule. + + This function performs two main tasks: + 1. Removes instrument configurations that are not present in the schedule. + 2. Verifies that all instruments in the schedule have corresponding configurations. + + Parameters + ---------- + schedule : Schedule + The schedule object containing the planned instruments and waypoints. + + Returns + ------- + None + + Raises + ------ + ConfigError + If an instrument in the schedule does not have a corresponding configuration. + + Notes + ----- + - Prints a message if a configuration is provided for an instrument not in the schedule. + - Sets the configuration to None for instruments not in the schedule. + - Raises a ConfigError for each instrument in the schedule that lacks a configuration. + + """ + instruments_in_schedule = schedule.get_instruments() + + for instrument in [ + "ARGO_FLOAT", + "DRIFTER", + "XBT", + "CTD", + "CTD_BGC", + ]: # TODO make instrument names consistent capitals or lowercase throughout codebase + if hasattr(self, instrument.lower() + "_config") and not any( + instrument == schedule_instrument.name + for schedule_instrument in instruments_in_schedule + ): + print(f"{instrument} configuration provided but not in schedule.") + setattr(self, instrument.lower() + "_config", None) + + # verify instruments in schedule have configuration + # TODO: the ConfigError message could be improved to explain that the **schedule** file has X instrument but the **ship_config** file does not + for instrument in instruments_in_schedule: + try: + InstrumentType(instrument) + except ValueError as e: + raise NotImplementedError("Instrument not supported.") from e + + if instrument == InstrumentType.ARGO_FLOAT and ( + not hasattr(self, "argo_float_config") or self.argo_float_config is None + ): + raise ConfigError( + "Planning has a waypoint with Argo float instrument, but configuration does not configure Argo floats." + ) + if instrument == InstrumentType.CTD and ( + not hasattr(self, "ctd_config") or self.ctd_config is None + ): + raise ConfigError( + "Planning has a waypoint with CTD instrument, but configuration does not configure CTDs." + ) + if instrument == InstrumentType.CTD_BGC and ( + not hasattr(self, "ctd_bgc_config") or self.ctd_bgc_config is None + ): + raise ConfigError( + "Planning has a waypoint with CTD_BGC instrument, but configuration does not configure CTD_BGCs." + ) + if instrument == InstrumentType.DRIFTER and ( + not hasattr(self, "drifter_config") or self.drifter_config is None + ): + raise ConfigError( + "Planning has a waypoint with drifter instrument, but configuration does not configure drifters." + ) + + if instrument == InstrumentType.XBT and ( + not hasattr(self, "xbt_config") or self.xbt_config is None + ): + raise ConfigError( + "Planning has a waypoint with XBT instrument, but configuration does not configure XBT." + ) From 851dd270c8cce9faeb8afdac6ebe0dfeda516a73 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 17 Sep 2025 15:22:46 +0100 Subject: [PATCH 07/97] add call to download_data() --- src/virtualship/cli/_fetch.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 127ee256..094f8c8a 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -110,14 +110,14 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None ) # keep only instruments in INTSTRUMENTS which are in schedule - filtered_instruments = { + filter_instruments = { k: v for k, v in INSTRUMENTS.items() if k in instruments_in_schedule } # iterate across instruments and download data based on space_time_region - for _, instrument in filtered_instruments.items(): + for _, instrument in filter_instruments.items(): try: - instrument["input_class"]( + input_dataset = instrument["input_class"]( data_dir=download_folder, credentials=credentials, space_time_region=space_time_region, From a417c68b350fe055e76c14cbc93ed95424787571 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 6 Oct 2025 11:32:52 +0200 Subject: [PATCH 08/97] Add new instrument classes and update InputDataset to include depth parameters --- src/virtualship/cli/_fetch.py | 5 +- src/virtualship/instruments/adcp.py | 70 ++++ src/virtualship/instruments/argo_float.py | 84 +++++ src/virtualship/instruments/ctd.py | 8 +- src/virtualship/instruments/ctd_bgc.py | 326 +++++------------- src/virtualship/instruments/drifter.py | 79 +++++ src/virtualship/instruments/master.py | 38 +- .../instruments/ship_underwater_st.py | 75 ++++ src/virtualship/instruments/xbt.py | 84 +++++ src/virtualship/models/instruments.py | 13 +- 10 files changed, 491 insertions(+), 291 deletions(-) create mode 100644 src/virtualship/instruments/adcp.py create mode 100644 src/virtualship/instruments/argo_float.py create mode 100644 src/virtualship/instruments/drifter.py create mode 100644 src/virtualship/instruments/ship_underwater_st.py create mode 100644 src/virtualship/instruments/xbt.py diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 094f8c8a..1979b153 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -89,7 +89,10 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None ) shutil.copyfile(path / EXPEDITION, download_folder / EXPEDITION) - # bathymetry (required for all expeditions) + # bathymetry + # TODO: this logic means it is downloaded for all expeditions but is only needed for CTD, CTD_BGC and XBT... + # TODO: to discuss: fine to still download for all expeditions because small size and then less duplication + # TODO: or add as var in each of InputDataset objects per instrument because will be overwritten to disk anyway and therefore not duplicate? copernicusmarine.subset( dataset_id="cmems_mod_glo_phy_my_0.083deg_static", variables=["deptho"], diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py new file mode 100644 index 00000000..71d8e2af --- /dev/null +++ b/src/virtualship/instruments/adcp.py @@ -0,0 +1,70 @@ +from dataclasses import dataclass +from typing import ClassVar + +from virtualship.models import instruments + +## TODO: __init__.py will also need updating! +# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py + + +@dataclass +class ADCP: + """ADCP configuration.""" + + name: ClassVar[str] = "ADCP" + + +# --------------- +# TODO: KERNELS +# --------------- + + +class ADCPInputDataset(instruments.InputDataset): + """Input dataset for ADCP instrument.""" + + DOWNLOAD_BUFFERS: ClassVar[dict] = { + "latlon_degrees": 0.0, + "days": 0.0, + } # ADCP data requires no buffers + + DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} + + def __init__(self, data_dir, credentials, space_time_region): + """Initialise with instrument's name.""" + super().__init__( + ADCP.name, + self.DOWNLOAD_BUFFERS["latlon_degrees"], + self.DOWNLOAD_BUFFERS["days"], + space_time_region.spatial_range.minimum_depth, + space_time_region.spatial_range.maximum_depth, + data_dir, + credentials, + space_time_region, + ) + + def get_datasets_dict(self) -> dict: + """Get variable specific args for instrument.""" + return { + "UVdata": { + "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "variables": ["uo", "vo"], + "output_filename": f"{self.name}_uv.nc", + }, + } + + +class ADCPInstrument(instruments.Instrument): + """ADCP instrument class.""" + + def __init__( + self, + config, + input_dataset, + kernels, + ): + """Initialise with instrument's name.""" + super().__init__(ADCP.name, config, input_dataset, kernels) + + def simulate(self): + """Simulate measurements.""" + ... diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py new file mode 100644 index 00000000..38eae990 --- /dev/null +++ b/src/virtualship/instruments/argo_float.py @@ -0,0 +1,84 @@ +from dataclasses import dataclass +from datetime import timedelta +from typing import ClassVar + +from virtualship.models import Spacetime, instruments + +## TODO: __init__.py will also need updating! +# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py + + +@dataclass +class ArgoFloat: + """Argo float configuration.""" + + name: ClassVar[str] = "ArgoFloat" + spacetime: Spacetime + depth: float # depth at which it floats and samples + lifetime: timedelta | None # if none, lifetime is infinite + + +# --------------- +# TODO: KERNELS +# --------------- + + +class ArgoFloatInputDataset(instruments.InputDataset): + """Input dataset for ArgoFloat instrument.""" + + DOWNLOAD_BUFFERS: ClassVar[dict] = { + "latlon_degrees": 3.0, + "days": 21.0, + } + + DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} + + def __init__(self, data_dir, credentials, space_time_region): + """Initialise with instrument's name.""" + super().__init__( + ArgoFloat.name, + self.DOWNLOAD_BUFFERS["latlon_degrees"], + self.DOWNLOAD_BUFFERS["days"], + self.DOWNLOAD_LIMITS["min_depth"], + space_time_region.spatial_range.maximum_depth, + data_dir, + credentials, + space_time_region, + ) + + def get_datasets_dict(self) -> dict: + """Get variable specific args for instrument.""" + return { + "UVdata": { + "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "variables": ["uo", "vo"], + "output_filename": "argo_float_uv.nc", + }, + "Sdata": { + "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "variables": ["so"], + "output_filename": "argo_float_s.nc", + }, + "Tdata": { + "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "variables": ["thetao"], + "output_filename": "argo_float_t.nc", + }, + } + + +class ArgoFloatInstrument(instruments.Instrument): + """ArgoFloat instrument class.""" + + def __init__( + self, + config, + input_dataset, + kernels, + ): + """Initialise with instrument's name.""" + super().__init__(ArgoFloat.name, config, input_dataset, kernels) + + def simulate(self): + """Simulate measurements.""" + ... diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 624256cb..09813241 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -36,6 +36,8 @@ def __init__(self, data_dir, credentials, space_time_region): CTD.name, self.DOWNLOAD_BUFFERS["latlon_degrees"], self.DOWNLOAD_BUFFERS["days"], + space_time_region.spatial_range.minimum_depth, + space_time_region.spatial_range.maximum_depth, data_dir, credentials, space_time_region, @@ -72,9 +74,3 @@ def __init__( def simulate(self): """Simulate measurements.""" ... - - -# # [PSEUDO-CODE] example implementation for reference -# ctd = CTDInstrument(config=CTD, data_dir=..., kernels=...) - -# ctd.simulate(...) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 6b9b2f29..77be7a06 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -1,263 +1,103 @@ -# from dataclasses import dataclass -# from typing import ClassVar - -# from virtualship.models import Spacetime, instruments - -# MYINSTRUMENT = "CTD_BGC" - - -# @dataclass -# class CTD_BGC: -# """CTD_BGC configuration.""" - -# spacetime: Spacetime -# min_depth: float -# max_depth: float - - -# # --------------- -# # TODO: KERNELS -# # --------------- - - -# class CTD_BGCInputDataset(instruments.InputDataset): -# """Input dataset object for CTD_BGC instrument.""" - -# DOWNLOAD_BUFFERS: ClassVar[dict] = { -# "latlon_degrees": 0.0, -# "days": 0.0, -# } # CTD_BGC data requires no buffers - -# def __init__(self, data_dir, credentials, space_time_region): -# """Initialise with instrument's name.""" -# super().__init__( -# MYINSTRUMENT, -# self.DOWNLOAD_BUFFERS["latlon_degrees"], -# self.DOWNLOAD_BUFFERS["days"], -# data_dir, -# credentials, -# space_time_region, -# ) - -# def datasets_dir(self) -> dict: -# """Variable specific args for instrument.""" -# return { -# "o2data": { -# "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", -# "variables": ["o2"], -# "output_filename": "ctd_bgc_o2.nc", -# }, -# "chlorodata": { -# "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", -# "variables": ["chl"], -# "output_filename": "ctd_bgc_chloro.nc", -# }, -# } - - -# class CTD_BGCInstrument(instruments.Instrument): -# """CTD_BGC instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(MYINSTRUMENT, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -# # # [PSEUDO-CODE] example implementation for reference -# # ctd = CTD_BGCInstrument(config=CTD_BGC, data_dir=..., kernels=...) - -# # ctd.simulate(...) - - -"""CTD_BGC instrument.""" - from dataclasses import dataclass -from datetime import timedelta -from pathlib import Path +from typing import ClassVar -import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable - -from virtualship.models import Spacetime +from virtualship.models import Spacetime, instruments @dataclass class CTD_BGC: - """Configuration for a single BGC CTD.""" + """CTD_BGC configuration.""" + name: ClassVar[str] = "CTD_BGC" spacetime: Spacetime min_depth: float max_depth: float -_CTD_BGCParticle = JITParticle.add_variables( - [ - Variable("o2", dtype=np.float32, initial=np.nan), - Variable("chl", dtype=np.float32, initial=np.nan), - Variable("no3", dtype=np.float32, initial=np.nan), - Variable("po4", dtype=np.float32, initial=np.nan), - Variable("ph", dtype=np.float32, initial=np.nan), - Variable("phyc", dtype=np.float32, initial=np.nan), - Variable("zooc", dtype=np.float32, initial=np.nan), - Variable("nppv", dtype=np.float32, initial=np.nan), - Variable("raising", dtype=np.int8, initial=0.0), # bool. 0 is False, 1 is True. - Variable("max_depth", dtype=np.float32), - Variable("min_depth", dtype=np.float32), - Variable("winch_speed", dtype=np.float32), - ] -) - - -def _sample_o2(particle, fieldset, time): - particle.o2 = fieldset.o2[time, particle.depth, particle.lat, particle.lon] +# --------------- +# TODO: KERNELS +# --------------- -def _sample_chlorophyll(particle, fieldset, time): - particle.chl = fieldset.chl[time, particle.depth, particle.lat, particle.lon] +class CTD_BGCInputDataset(instruments.InputDataset): + """Input dataset object for CTD_BGC instrument.""" + DOWNLOAD_BUFFERS: ClassVar[dict] = { + "latlon_degrees": 0.0, + "days": 0.0, + } # CTD_BGC data requires no buffers -def _sample_nitrate(particle, fieldset, time): - particle.no3 = fieldset.no3[time, particle.depth, particle.lat, particle.lon] - - -def _sample_phosphate(particle, fieldset, time): - particle.po4 = fieldset.po4[time, particle.depth, particle.lat, particle.lon] - - -def _sample_ph(particle, fieldset, time): - particle.ph = fieldset.ph[time, particle.depth, particle.lat, particle.lon] - - -def _sample_phytoplankton(particle, fieldset, time): - particle.phyc = fieldset.phyc[time, particle.depth, particle.lat, particle.lon] - - -def _sample_zooplankton(particle, fieldset, time): - particle.zooc = fieldset.zooc[time, particle.depth, particle.lat, particle.lon] - - -def _sample_primary_production(particle, fieldset, time): - particle.nppv = fieldset.nppv[time, particle.depth, particle.lat, particle.lon] - - -def _ctd_bgc_cast(particle, fieldset, time): - # lowering - if particle.raising == 0: - particle_ddepth = -particle.winch_speed * particle.dt - if particle.depth + particle_ddepth < particle.max_depth: - particle.raising = 1 - particle_ddepth = -particle_ddepth - # raising - else: - particle_ddepth = particle.winch_speed * particle.dt - if particle.depth + particle_ddepth > particle.min_depth: - particle.delete() - - -def simulate_ctd_bgc( - fieldset: FieldSet, - out_path: str | Path, - ctd_bgcs: list[CTD_BGC], - outputdt: timedelta, -) -> None: - """ - Use Parcels to simulate a set of BGC CTDs in a fieldset. - - :param fieldset: The fieldset to simulate the BGC CTDs in. - :param out_path: The path to write the results to. - :param ctds: A list of BGC CTDs to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :raises ValueError: Whenever provided BGC CTDs, fieldset, are not compatible with this function. - """ - WINCH_SPEED = 1.0 # sink and rise speed in m/s - DT = 10.0 # dt of CTD simulation integrator - - if len(ctd_bgcs) == 0: - print( - "No BGC CTDs provided. Parcels currently crashes when providing an empty particle set, so no BGC CTD simulation will be done and no files will be created." + def __init__(self, data_dir, credentials, space_time_region): + """Initialise with instrument's name.""" + super().__init__( + CTD_BGC.name, + self.DOWNLOAD_BUFFERS["latlon_degrees"], + self.DOWNLOAD_BUFFERS["days"], + space_time_region.spatial_range.minimum_depth, + space_time_region.spatial_range.maximum_depth, + data_dir, + credentials, + space_time_region, ) - # TODO when Parcels supports it this check can be removed. - return - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - - # deploy time for all ctds should be later than fieldset start time - if not all( - [ - np.datetime64(ctd_bgc.spacetime.time) >= fieldset_starttime - for ctd_bgc in ctd_bgcs - ] + def datasets_dir(self) -> dict: + """Variable specific args for instrument.""" + return { + "o2data": { + "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "variables": ["o2"], + "output_filename": "ctd_bgc_o2.nc", + }, + "chlorodata": { + "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "variables": ["chl"], + "output_filename": "ctd_bgc_chl.nc", + }, + "nitratedata": { + "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "variables": ["no3"], + "output_filename": "ctd_bgc_no3.nc", + }, + "phosphatedata": { + "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "variables": ["po4"], + "output_filename": "ctd_bgc_po4.nc", + }, + "phdata": { + "dataset_id": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", + "variables": ["ph"], + "output_filename": "ctd_bgc_ph.nc", + }, + "phytoplanktondata": { + "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "variables": ["phyc"], + "output_filename": "ctd_bgc_phyc.nc", + }, + "zooplanktondata": { + "dataset_id": "cmems_mod_glo_bgc-plankton_anfc_0.25deg_P1D-m", + "variables": ["zooc"], + "output_filename": "ctd_bgc_zooc.nc", + }, + "primaryproductiondata": { + "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "variables": ["nppv"], + "output_filename": "ctd_bgc_nppv.nc", + }, + } + + +class CTD_BGCInstrument(instruments.Instrument): + """CTD_BGC instrument class.""" + + def __init__( + self, + config, + input_dataset, + kernels, ): - raise ValueError("BGC CTD deployed before fieldset starts.") - - # depth the bgc ctd will go to. shallowest between bgc ctd max depth and bathymetry. - max_depths = [ - max( - ctd_bgc.max_depth, - fieldset.bathymetry.eval( - z=0, - y=ctd_bgc.spacetime.location.lat, - x=ctd_bgc.spacetime.location.lon, - time=0, - ), - ) - for ctd_bgc in ctd_bgcs - ] + """Initialise with instrument's name.""" + super().__init__(CTD_BGC.name, config, input_dataset, kernels) - # CTD depth can not be too shallow, because kernel would break. - # This shallow is not useful anyway, no need to support. - if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): - raise ValueError( - f"BGC CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" - ) - - # define parcel particles - ctd_bgc_particleset = ParticleSet( - fieldset=fieldset, - pclass=_CTD_BGCParticle, - lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in ctd_bgcs], - lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in ctd_bgcs], - depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], - time=[ctd_bgc.spacetime.time for ctd_bgc in ctd_bgcs], - max_depth=max_depths, - min_depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], - winch_speed=[WINCH_SPEED for _ in ctd_bgcs], - ) - - # define output file for the simulation - out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=outputdt) - - # execute simulation - ctd_bgc_particleset.execute( - [ - _sample_o2, - _sample_chlorophyll, - _sample_nitrate, - _sample_phosphate, - _sample_ph, - _sample_phytoplankton, - _sample_zooplankton, - _sample_primary_production, - _ctd_bgc_cast, - ], - endtime=fieldset_endtime, - dt=DT, - verbose_progress=False, - output_file=out_file, - ) - - # there should be no particles left, as they delete themselves when they resurface - if len(ctd_bgc_particleset.particledata) != 0: - raise ValueError( - "Simulation ended before BGC CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." - ) + def simulate(self): + """Simulate measurements.""" + ... diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py new file mode 100644 index 00000000..70ac4b7b --- /dev/null +++ b/src/virtualship/instruments/drifter.py @@ -0,0 +1,79 @@ +from dataclasses import dataclass +from datetime import timedelta +from typing import ClassVar + +from virtualship.models import Spacetime, instruments + +## TODO: __init__.py will also need updating! +# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py + + +@dataclass +class Drifter: + """Drifter configuration.""" + + name: ClassVar[str] = "Drifter" + spacetime: Spacetime + depth: float # depth at which it floats and samples + lifetime: timedelta | None # if none, lifetime is infinite + + +# --------------- +# TODO: KERNELS +# --------------- + + +class DrifterInputDataset(instruments.InputDataset): + """Input dataset for Drifter instrument.""" + + DOWNLOAD_BUFFERS: ClassVar[dict] = { + "latlon_degrees": 3.0, + "days": 21.0, + } + + DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1, "max_depth": 1} + + def __init__(self, data_dir, credentials, space_time_region): + """Initialise with instrument's name.""" + super().__init__( + Drifter.name, + self.DOWNLOAD_BUFFERS["latlon_degrees"], + self.DOWNLOAD_BUFFERS["days"], + self.DOWNLOAD_LIMITS["min_depth"], + self.DOWNLOAD_LIMITS["max_depth"], + data_dir, + credentials, + space_time_region, + ) + + def get_datasets_dict(self) -> dict: + """Get variable specific args for instrument.""" + return { + "UVdata": { + "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "variables": ["uo", "vo"], + "output_filename": "drifter_uv.nc", + }, + "Tdata": { + "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "variables": ["thetao"], + "output_filename": "drifter_t.nc", + }, + } + + +class DrifterInstrument(instruments.Instrument): + """Drifter instrument class.""" + + def __init__( + self, + config, + input_dataset, + kernels, + ): + """Initialise with instrument's name.""" + super().__init__(Drifter.name, config, input_dataset, kernels) + + def simulate(self): + """Simulate measurements.""" + ... diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index 4705c8e3..161348e2 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -15,29 +15,16 @@ class InstrumentType(Enum): """Types of the instruments.""" CTD = "CTD" - # CTD_BGC = "CTD_BGC" - # DRIFTER = "DRIFTER" - # ARGO_FLOAT = "ARGO_FLOAT" - # XBT = "XBT" + CTD_BGC = "CTD_BGC" + DRIFTER = "DRIFTER" + ARGO_FLOAT = "ARGO_FLOAT" + XBT = "XBT" # # TODO: should underway also be handled here?! # ADCP = "ADCP" # UNDERWAY_ST = "UNDERWAY_ST" -# replace with imports instead... -class CTDInputDataset: - """Input dataset class for CTD instrument.""" - - pass - - -class CTDInstrument: - """Instrument class for CTD instrument.""" - - pass - - INSTRUMENTS = { inst: { "input_class": globals()[f"{inst.value}InputDataset"], @@ -47,20 +34,3 @@ class CTDInstrument: if f"{inst.value}InputDataset" in globals() and f"{inst.value}Instrument" in globals() } - - -# INSTRUMENTS = { -# InstrumentType.CTD: { -# "input_class": CTDInputDataset, -# "instrument_class": CTDInstrument, -# } -# # and so on for other instruments... -# } - -# INSTRUMENTS = { -# "InstrumentType.CTD": { -# "input_class": "test", -# "instrument_class": "test", -# } -# # and so on for other instruments... -# } diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py new file mode 100644 index 00000000..f364b156 --- /dev/null +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -0,0 +1,75 @@ +from dataclasses import dataclass +from typing import ClassVar + +from virtualship.models import instruments + +## TODO: __init__.py will also need updating! +# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py + + +@dataclass +class Underwater_ST: + """Underwater_ST configuration.""" + + name: ClassVar[str] = "Underwater_ST" + + +# --------------- +# TODO: KERNELS +# --------------- + + +class Underwater_STInputDataset(instruments.InputDataset): + """Input dataset for Underwater_ST instrument.""" + + DOWNLOAD_BUFFERS: ClassVar[dict] = { + "latlon_degrees": 0.0, + "days": 0.0, + } # Underwater_ST data requires no buffers + + DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} + + def __init__(self, data_dir, credentials, space_time_region): + """Initialise with instrument's name.""" + super().__init__( + Underwater_ST.name, + self.DOWNLOAD_BUFFERS["latlon_degrees"], + self.DOWNLOAD_BUFFERS["days"], + -2.0, # is always at 2m depth + -2.0, # is always at 2m depth + data_dir, + credentials, + space_time_region, + ) + + def get_datasets_dict(self) -> dict: + """Get variable specific args for instrument.""" + return { + "Sdata": { + "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "variables": ["so"], + "output_filename": f"{self.name}_s.nc", + }, + "Tdata": { + "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "variables": ["thetao"], + "output_filename": f"{self.name}_t.nc", + }, + } + + +class Underwater_STInstrument(instruments.Instrument): + """Underwater_ST instrument class.""" + + def __init__( + self, + config, + input_dataset, + kernels, + ): + """Initialise with instrument's name.""" + super().__init__(Underwater_ST.name, config, input_dataset, kernels) + + def simulate(self): + """Simulate measurements.""" + ... diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py new file mode 100644 index 00000000..dcec018b --- /dev/null +++ b/src/virtualship/instruments/xbt.py @@ -0,0 +1,84 @@ +from dataclasses import dataclass +from datetime import timedelta +from typing import ClassVar + +from virtualship.models import Spacetime, instruments + +## TODO: __init__.py will also need updating! +# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py + + +@dataclass +class XBT: + """XBT configuration.""" + + name: ClassVar[str] = "XBT" + spacetime: Spacetime + depth: float # depth at which it floats and samples + lifetime: timedelta | None # if none, lifetime is infinite + + +# --------------- +# TODO: KERNELS +# --------------- + + +class XBTInputDataset(instruments.InputDataset): + """Input dataset for XBT instrument.""" + + DOWNLOAD_BUFFERS: ClassVar[dict] = { + "latlon_degrees": 3.0, + "days": 21.0, + } + + DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} + + def __init__(self, data_dir, credentials, space_time_region): + """Initialise with instrument's name.""" + super().__init__( + XBT.name, + self.DOWNLOAD_BUFFERS["latlon_degrees"], + self.DOWNLOAD_BUFFERS["days"], + self.DOWNLOAD_LIMITS["min_depth"], + space_time_region.spatial_range.maximum_depth, + data_dir, + credentials, + space_time_region, + ) + + def get_datasets_dict(self) -> dict: + """Get variable specific args for instrument.""" + return { + "UVdata": { + "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "variables": ["uo", "vo"], + "output_filename": "ship_uv.nc", + }, + "Sdata": { + "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "variables": ["so"], + "output_filename": "ship_s.nc", + }, + "Tdata": { + "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "variables": ["thetao"], + "output_filename": "ship_t.nc", + }, + } + + +class XBTInstrument(instruments.Instrument): + """XBT instrument class.""" + + def __init__( + self, + config, + input_dataset, + kernels, + ): + """Initialise with instrument's name.""" + super().__init__(XBT.name, config, input_dataset, kernels) + + def simulate(self): + """Simulate measurements.""" + ... diff --git a/src/virtualship/models/instruments.py b/src/virtualship/models/instruments.py index cddebd85..57139add 100644 --- a/src/virtualship/models/instruments.py +++ b/src/virtualship/models/instruments.py @@ -31,6 +31,8 @@ def __init__( name: str, latlon_buffer: float, datetime_buffer: float, + min_depth: float, + max_depth: float, data_dir: str, credentials: dict, space_time_region: SpaceTimeRegion, @@ -39,6 +41,8 @@ def __init__( self.name = name self.latlon_buffer = latlon_buffer self.datetime_buffer = datetime_buffer + self.min_depth = min_depth + self.max_depth = max_depth self.data_dir = data_dir self.credentials = credentials self.space_time_region = space_time_region @@ -62,8 +66,8 @@ def download_data(self) -> None: start_datetime=self.space_time_region.time_range.start_time, end_datetime=self.space_time_region.time_range.end_time + timedelta(days=self.datetime_buffer), - minimum_depth=abs(self.space_time_region.spatial_range.minimum_depth), - maximum_depth=abs(self.space_time_region.spatial_range.maximum_depth), + minimum_depth=abs(self.min_depth), + maximum_depth=abs(self.max_depth), output_directory=self.data_dir, username=self.credentials["username"], password=self.credentials["password"], @@ -77,11 +81,6 @@ def download_data(self) -> None: download_args = {**parameter_args, **dataset} copernicusmarine.subset(**download_args) - # def get_fieldset_paths(self) -> list: - # """List of paths for instrument's (downloaded) input data.""" - - # ... - class Instrument(abc.ABC): """Base class for instruments.""" From a0c77abe91ae43f32889a39a1c26b0bfd0d178ca Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 7 Oct 2025 11:52:28 +0200 Subject: [PATCH 09/97] Refactor instrument handling in _fetch and update imports for consistency --- src/virtualship/cli/_fetch.py | 21 +++++++++- src/virtualship/instruments/master.py | 56 ++++++++++++++++----------- src/virtualship/models/instruments.py | 7 ---- src/virtualship/models/schedule.py | 2 +- src/virtualship/models/ship_config.py | 2 +- 5 files changed, 54 insertions(+), 34 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 1979b153..c258f044 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -25,9 +25,12 @@ import virtualship.cli._creds as creds from virtualship.utils import EXPEDITION from virtualship.instruments.master import INSTRUMENTS +from virtualship.instruments.master import InstrumentType, get_instruments_registry DOWNLOAD_METADATA = "download_metadata.yaml" +INSTRUMENTS = get_instruments_registry() + def _fetch(path: str | Path, username: str | None, password: str | None) -> None: """ @@ -81,6 +84,20 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None end_datetime = time_range.end_time instruments_in_schedule = expedition.schedule.get_instruments() + # TEMPORARY measure to get underway instruments in `instruments_in_schedule` + # TODO: should evaporate when schedule and ship_config.yaml files are consolidated in a separate PR... + if ship_config.adcp_config is not None: + instruments_in_schedule.add(InstrumentType.ADCP) + if ship_config.ship_underwater_st_config is not None: + instruments_in_schedule.add(InstrumentType.UNDERWATER_ST) + + # TEMPORARY measure to get underway instruments in `instruments_in_schedule` + # TODO: should evaporate when schedule and ship_config.yaml files are consolidated in a separate PR... + if ship_config.adcp_config is not None: + instruments_in_schedule.add(InstrumentType.ADCP) + if ship_config.ship_underwater_st_config is not None: + instruments_in_schedule.add(InstrumentType.UNDERWATER_ST) + # Create download folder and set download metadata download_folder = data_dir / hash_to_filename(space_time_region_hash) download_folder.mkdir() @@ -118,7 +135,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None } # iterate across instruments and download data based on space_time_region - for _, instrument in filter_instruments.items(): + for itype, instrument in filter_instruments.items(): try: input_dataset = instrument["input_class"]( data_dir=download_folder, @@ -237,7 +254,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None shutil.rmtree(download_folder) raise e - click.echo(f"{instrument.name} data download completed.") # TODO + click.echo(f"{itype.value} data download completed.") complete_download(download_folder) diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index 161348e2..58eb0591 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -1,36 +1,46 @@ -# - -# TODO: temporary measure so as not to have to overhaul the InstrumentType class logic in one go -#! And also to avoid breaking other parts of the codebase which rely on InstrumentType when for now just working on fetch -# TODO: ideally this can evaporate... -# TODO: discuss to see if there's a better option...! - from enum import Enum -# and so on ... -# from virtualship.instruments.ctd import CTDInputDataset, CTDInstrument - class InstrumentType(Enum): """Types of the instruments.""" + # TODO: temporary measure so as not to have to overhaul the InstrumentType class logic in one go + #! And also to avoid breaking other parts of the codebase which rely on InstrumentType when for now just working on fetch + # TODO: ideally this can evaporate in a future PR... + CTD = "CTD" CTD_BGC = "CTD_BGC" DRIFTER = "DRIFTER" ARGO_FLOAT = "ARGO_FLOAT" XBT = "XBT" + ADCP = "ADCP" + UNDERWATER_ST = "UNDERWATER_ST" + + +def get_instruments_registry(): + # local imports to avoid circular import issues + from virtualship.instruments.adcp import ADCPInputDataset + from virtualship.instruments.argo_float import ArgoFloatInputDataset + from virtualship.instruments.ctd import CTDInputDataset + from virtualship.instruments.ctd_bgc import CTD_BGCInputDataset + from virtualship.instruments.drifter import DrifterInputDataset + from virtualship.instruments.ship_underwater_st import Underwater_STInputDataset + from virtualship.instruments.xbt import XBTInputDataset + + _input_class_map = { + "CTD": CTDInputDataset, + "CTD_BGC": CTD_BGCInputDataset, + "DRIFTER": DrifterInputDataset, + "ARGO_FLOAT": ArgoFloatInputDataset, + "XBT": XBTInputDataset, + "ADCP": ADCPInputDataset, + "UNDERWATER_ST": Underwater_STInputDataset, + } - # # TODO: should underway also be handled here?! - # ADCP = "ADCP" - # UNDERWAY_ST = "UNDERWAY_ST" - - -INSTRUMENTS = { - inst: { - "input_class": globals()[f"{inst.value}InputDataset"], - "instrument_class": globals()[f"{inst.value}Instrument"], + return { + inst: { + "input_class": _input_class_map.get(inst.value), + } + for inst in InstrumentType + if _input_class_map.get(inst.value) is not None } - for inst in InstrumentType - if f"{inst.value}InputDataset" in globals() - and f"{inst.value}Instrument" in globals() -} diff --git a/src/virtualship/models/instruments.py b/src/virtualship/models/instruments.py index 57139add..7a356290 100644 --- a/src/virtualship/models/instruments.py +++ b/src/virtualship/models/instruments.py @@ -121,10 +121,3 @@ def run(self): def simulate(self): """Simulate instrument measurements.""" ... - - -# e.g. pseudo-code ... -# TODO: (necessary?) how to dynamically assemble list of all instruments defined so that new instruments can be added only by changes in one place...? -available_instruments: list = ... -# for instrument in available_instruments: -# MyInstrument(instrument) diff --git a/src/virtualship/models/schedule.py b/src/virtualship/models/schedule.py index 091c23b4..f3e5dabe 100644 --- a/src/virtualship/models/schedule.py +++ b/src/virtualship/models/schedule.py @@ -12,8 +12,8 @@ import yaml from virtualship.errors import ScheduleError +from virtualship.instruments.master import InstrumentType -from .instruments import InstrumentType from .location import Location from .space_time_region import SpaceTimeRegion diff --git a/src/virtualship/models/ship_config.py b/src/virtualship/models/ship_config.py index 61d3d390..ba7d221f 100644 --- a/src/virtualship/models/ship_config.py +++ b/src/virtualship/models/ship_config.py @@ -10,7 +10,7 @@ import yaml from virtualship.errors import ConfigError -from virtualship.models.instruments import InstrumentType +from virtualship.instruments.master import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta if TYPE_CHECKING: From 9bcaba2864566b02b5f8ec4ceeba463b2152aba0 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 7 Oct 2025 11:53:34 +0200 Subject: [PATCH 10/97] Refactor instrument classes and re-add (temporary) simulation functions across multiple files --- src/virtualship/instruments/adcp.py | 111 ++++++++-- src/virtualship/instruments/argo_float.py | 200 ++++++++++++++++-- src/virtualship/instruments/ctd.py | 148 +++++++++++-- src/virtualship/instruments/ctd_bgc.py | 196 +++++++++++++++-- src/virtualship/instruments/drifter.py | 129 +++++++++-- .../instruments/ship_underwater_st.py | 102 +++++++-- src/virtualship/instruments/xbt.py | 151 +++++++++++-- 7 files changed, 917 insertions(+), 120 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 71d8e2af..052004fb 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -1,7 +1,11 @@ from dataclasses import dataclass +from pathlib import Path from typing import ClassVar -from virtualship.models import instruments +import numpy as np +from parcels import FieldSet, ParticleSet, ScipyParticle, Variable + +from virtualship.models import Spacetime, instruments ## TODO: __init__.py will also need updating! # + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py @@ -14,9 +18,20 @@ class ADCP: name: ClassVar[str] = "ADCP" -# --------------- -# TODO: KERNELS -# --------------- +# we specifically use ScipyParticle because we have many small calls to execute +# there is some overhead with JITParticle and this ends up being significantly faster +_ADCPParticle = ScipyParticle.add_variables( + [ + Variable("U", dtype=np.float32, initial=np.nan), + Variable("V", dtype=np.float32, initial=np.nan), + ] +) + + +def _sample_velocity(particle, fieldset, time): + particle.U, particle.V = fieldset.UV.eval( + time, particle.depth, particle.lat, particle.lon, applyConversion=False + ) class ADCPInputDataset(instruments.InputDataset): @@ -53,18 +68,78 @@ def get_datasets_dict(self) -> dict: } -class ADCPInstrument(instruments.Instrument): - """ADCP instrument class.""" - - def __init__( - self, - config, - input_dataset, - kernels, - ): - """Initialise with instrument's name.""" - super().__init__(ADCP.name, config, input_dataset, kernels) +# TODO: uncomment when ready for new simulation logic! +# class ADCPInstrument(instruments.Instrument): +# """ADCP instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(ADCP.name, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +# TODO: to be replaced with new simulation logic +## -- old simulation code + + +def simulate_adcp( + fieldset: FieldSet, + out_path: str | Path, + max_depth: float, + min_depth: float, + num_bins: int, + sample_points: list[Spacetime], +) -> None: + """ + Use Parcels to simulate an ADCP in a fieldset. + + :param fieldset: The fieldset to simulate the ADCP in. + :param out_path: The path to write the results to. + :param max_depth: Maximum depth the ADCP can measure. + :param min_depth: Minimum depth the ADCP can measure. + :param num_bins: How many samples to take in the complete range between max_depth and min_depth. + :param sample_points: The places and times to sample at. + """ + sample_points.sort(key=lambda p: p.time) + + bins = np.linspace(max_depth, min_depth, num_bins) + num_particles = len(bins) + particleset = ParticleSet.from_list( + fieldset=fieldset, + pclass=_ADCPParticle, + lon=np.full( + num_particles, 0.0 + ), # initial lat/lon are irrelevant and will be overruled later. + lat=np.full(num_particles, 0.0), + depth=bins, + time=0, # same for time + ) + + # define output file for the simulation + # outputdt set to infinite as we just want to write at the end of every call to 'execute' + out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) + + for point in sample_points: + particleset.lon_nextloop[:] = point.location.lon + particleset.lat_nextloop[:] = point.location.lat + particleset.time_nextloop[:] = fieldset.time_origin.reltime( + np.datetime64(point.time) + ) - def simulate(self): - """Simulate measurements.""" - ... + # perform one step using the particleset + # dt and runtime are set so exactly one step is made. + particleset.execute( + [_sample_velocity], + dt=1, + runtime=1, + verbose_progress=False, + output_file=out_file, + ) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 38eae990..4769eccf 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -1,7 +1,19 @@ +import math from dataclasses import dataclass -from datetime import timedelta +from datetime import datetime, timedelta +from pathlib import Path from typing import ClassVar +import numpy as np +from parcels import ( + AdvectionRK4, + FieldSet, + JITParticle, + ParticleSet, + StatusCode, + Variable, +) + from virtualship.models import Spacetime, instruments ## TODO: __init__.py will also need updating! @@ -18,9 +30,88 @@ class ArgoFloat: lifetime: timedelta | None # if none, lifetime is infinite -# --------------- -# TODO: KERNELS -# --------------- +_ArgoParticle = JITParticle.add_variables( + [ + Variable("cycle_phase", dtype=np.int32, initial=0.0), + Variable("cycle_age", dtype=np.float32, initial=0.0), + Variable("drift_age", dtype=np.float32, initial=0.0), + Variable("salinity", dtype=np.float32, initial=np.nan), + Variable("temperature", dtype=np.float32, initial=np.nan), + Variable("min_depth", dtype=np.float32), + Variable("max_depth", dtype=np.float32), + Variable("drift_depth", dtype=np.float32), + Variable("vertical_speed", dtype=np.float32), + Variable("cycle_days", dtype=np.int32), + Variable("drift_days", dtype=np.int32), + ] +) + + +def _argo_float_vertical_movement(particle, fieldset, time): + if particle.cycle_phase == 0: + # Phase 0: Sinking with vertical_speed until depth is drift_depth + particle_ddepth += ( # noqa Parcels defines particle_* variables, which code checkers cannot know. + particle.vertical_speed * particle.dt + ) + if particle.depth + particle_ddepth <= particle.drift_depth: + particle_ddepth = particle.drift_depth - particle.depth + particle.cycle_phase = 1 + + elif particle.cycle_phase == 1: + # Phase 1: Drifting at depth for drifttime seconds + particle.drift_age += particle.dt + if particle.drift_age >= particle.drift_days * 86400: + particle.drift_age = 0 # reset drift_age for next cycle + particle.cycle_phase = 2 + + elif particle.cycle_phase == 2: + # Phase 2: Sinking further to max_depth + particle_ddepth += particle.vertical_speed * particle.dt + if particle.depth + particle_ddepth <= particle.max_depth: + particle_ddepth = particle.max_depth - particle.depth + particle.cycle_phase = 3 + + elif particle.cycle_phase == 3: + # Phase 3: Rising with vertical_speed until at surface + particle_ddepth -= particle.vertical_speed * particle.dt + particle.cycle_age += ( + particle.dt + ) # solve issue of not updating cycle_age during ascent + if particle.depth + particle_ddepth >= particle.min_depth: + particle_ddepth = particle.min_depth - particle.depth + particle.temperature = ( + math.nan + ) # reset temperature to NaN at end of sampling cycle + particle.salinity = math.nan # idem + particle.cycle_phase = 4 + else: + particle.temperature = fieldset.T[ + time, particle.depth, particle.lat, particle.lon + ] + particle.salinity = fieldset.S[ + time, particle.depth, particle.lat, particle.lon + ] + + elif particle.cycle_phase == 4: + # Phase 4: Transmitting at surface until cycletime is reached + if particle.cycle_age > particle.cycle_days * 86400: + particle.cycle_phase = 0 + particle.cycle_age = 0 + + if particle.state == StatusCode.Evaluate: + particle.cycle_age += particle.dt # update cycle_age + + +def _keep_at_surface(particle, fieldset, time): + # Prevent error when float reaches surface + if particle.state == StatusCode.ErrorThroughSurface: + particle.depth = particle.min_depth + particle.state = StatusCode.Success + + +def _check_error(particle, fieldset, time): + if particle.state >= 50: # This captures all Errors + particle.delete() class ArgoFloatInputDataset(instruments.InputDataset): @@ -67,18 +158,89 @@ def get_datasets_dict(self) -> dict: } -class ArgoFloatInstrument(instruments.Instrument): - """ArgoFloat instrument class.""" - - def __init__( - self, - config, - input_dataset, - kernels, - ): - """Initialise with instrument's name.""" - super().__init__(ArgoFloat.name, config, input_dataset, kernels) - - def simulate(self): - """Simulate measurements.""" - ... +# class ArgoFloatInstrument(instruments.Instrument): +# """ArgoFloat instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(ArgoFloat.name, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +def simulate_argo_floats( + fieldset: FieldSet, + out_path: str | Path, + argo_floats: list[ArgoFloat], + outputdt: timedelta, + endtime: datetime | None, +) -> None: + """ + Use Parcels to simulate a set of Argo floats in a fieldset. + + :param fieldset: The fieldset to simulate the Argo floats in. + :param out_path: The path to write the results to. + :param argo_floats: A list of Argo floats to simulate. + :param outputdt: Interval which dictates the update frequency of file output during simulation + :param endtime: Stop at this time, or if None, continue until the end of the fieldset. + """ + DT = 10.0 # dt of Argo float simulation integrator + + if len(argo_floats) == 0: + print( + "No Argo floats provided. Parcels currently crashes when providing an empty particle set, so no argo floats simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + # define parcel particles + argo_float_particleset = ParticleSet( + fieldset=fieldset, + pclass=_ArgoParticle, + lat=[argo.spacetime.location.lat for argo in argo_floats], + lon=[argo.spacetime.location.lon for argo in argo_floats], + depth=[argo.min_depth for argo in argo_floats], + time=[argo.spacetime.time for argo in argo_floats], + min_depth=[argo.min_depth for argo in argo_floats], + max_depth=[argo.max_depth for argo in argo_floats], + drift_depth=[argo.drift_depth for argo in argo_floats], + vertical_speed=[argo.vertical_speed for argo in argo_floats], + cycle_days=[argo.cycle_days for argo in argo_floats], + drift_days=[argo.drift_days for argo in argo_floats], + ) + + # define output file for the simulation + out_file = argo_float_particleset.ParticleFile( + name=out_path, outputdt=outputdt, chunks=[len(argo_float_particleset), 100] + ) + + # get earliest between fieldset end time and provide end time + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + if endtime is None: + actual_endtime = fieldset_endtime + elif endtime > fieldset_endtime: + print("WARN: Requested end time later than fieldset end time.") + actual_endtime = fieldset_endtime + else: + actual_endtime = np.timedelta64(endtime) + + # execute simulation + argo_float_particleset.execute( + [ + _argo_float_vertical_movement, + AdvectionRK4, + _keep_at_surface, + _check_error, + ], + endtime=actual_endtime, + dt=DT, + output_file=out_file, + verbose_progress=True, + ) diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 09813241..070cdb78 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -1,6 +1,11 @@ from dataclasses import dataclass +from datetime import timedelta +from pathlib import Path from typing import ClassVar +import numpy as np +from parcels import FieldSet, JITParticle, ParticleSet, Variable + from virtualship.models import Spacetime, instruments ## TODO: __init__.py will also need updating! @@ -17,9 +22,38 @@ class CTD: max_depth: float -# --------------- -# TODO: KERNELS -# --------------- +_CTDParticle = JITParticle.add_variables( + [ + Variable("salinity", dtype=np.float32, initial=np.nan), + Variable("temperature", dtype=np.float32, initial=np.nan), + Variable("raising", dtype=np.int8, initial=0.0), # bool. 0 is False, 1 is True. + Variable("max_depth", dtype=np.float32), + Variable("min_depth", dtype=np.float32), + Variable("winch_speed", dtype=np.float32), + ] +) + + +def _sample_temperature(particle, fieldset, time): + particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] + + +def _sample_salinity(particle, fieldset, time): + particle.salinity = fieldset.S[time, particle.depth, particle.lat, particle.lon] + + +def _ctd_cast(particle, fieldset, time): + # lowering + if particle.raising == 0: + particle_ddepth = -particle.winch_speed * particle.dt + if particle.depth + particle_ddepth < particle.max_depth: + particle.raising = 1 + particle_ddepth = -particle_ddepth + # raising + else: + particle_ddepth = particle.winch_speed * particle.dt + if particle.depth + particle_ddepth > particle.min_depth: + particle.delete() class CTDInputDataset(instruments.InputDataset): @@ -59,18 +93,102 @@ def get_datasets_dict(self) -> dict: } -class CTDInstrument(instruments.Instrument): - """CTD instrument class.""" +# class CTDInstrument(instruments.Instrument): +# """CTD instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(CTD.name, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +def simulate_ctd( + fieldset: FieldSet, + out_path: str | Path, + ctds: list[CTD], + outputdt: timedelta, +) -> None: + """ + Use Parcels to simulate a set of CTDs in a fieldset. + + :param fieldset: The fieldset to simulate the CTDs in. + :param out_path: The path to write the results to. + :param ctds: A list of CTDs to simulate. + :param outputdt: Interval which dictates the update frequency of file output during simulation + :raises ValueError: Whenever provided CTDs, fieldset, are not compatible with this function. + """ + WINCH_SPEED = 1.0 # sink and rise speed in m/s + DT = 10.0 # dt of CTD simulation integrator + + if len(ctds) == 0: + print( + "No CTDs provided. Parcels currently crashes when providing an empty particle set, so no CTD simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - def __init__( - self, - config, - input_dataset, - kernels, + # deploy time for all ctds should be later than fieldset start time + if not all( + [np.datetime64(ctd.spacetime.time) >= fieldset_starttime for ctd in ctds] ): - """Initialise with instrument's name.""" - super().__init__(CTD.name, config, input_dataset, kernels) + raise ValueError("CTD deployed before fieldset starts.") + + # depth the ctd will go to. shallowest between ctd max depth and bathymetry. + max_depths = [ + max( + ctd.max_depth, + fieldset.bathymetry.eval( + z=0, y=ctd.spacetime.location.lat, x=ctd.spacetime.location.lon, time=0 + ), + ) + for ctd in ctds + ] + + # CTD depth can not be too shallow, because kernel would break. + # This shallow is not useful anyway, no need to support. + if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): + raise ValueError( + f"CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" + ) - def simulate(self): - """Simulate measurements.""" - ... + # define parcel particles + ctd_particleset = ParticleSet( + fieldset=fieldset, + pclass=_CTDParticle, + lon=[ctd.spacetime.location.lon for ctd in ctds], + lat=[ctd.spacetime.location.lat for ctd in ctds], + depth=[ctd.min_depth for ctd in ctds], + time=[ctd.spacetime.time for ctd in ctds], + max_depth=max_depths, + min_depth=[ctd.min_depth for ctd in ctds], + winch_speed=[WINCH_SPEED for _ in ctds], + ) + + # define output file for the simulation + out_file = ctd_particleset.ParticleFile(name=out_path, outputdt=outputdt) + + # execute simulation + ctd_particleset.execute( + [_sample_salinity, _sample_temperature, _ctd_cast], + endtime=fieldset_endtime, + dt=DT, + verbose_progress=False, + output_file=out_file, + ) + + # there should be no particles left, as they delete themselves when they resurface + if len(ctd_particleset.particledata) != 0: + raise ValueError( + "Simulation ended before CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." + ) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 77be7a06..1025a5c8 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -1,6 +1,11 @@ from dataclasses import dataclass +from datetime import timedelta +from pathlib import Path from typing import ClassVar +import numpy as np +from parcels import FieldSet, JITParticle, ParticleSet, Variable + from virtualship.models import Spacetime, instruments @@ -14,9 +19,68 @@ class CTD_BGC: max_depth: float -# --------------- -# TODO: KERNELS -# --------------- +_CTD_BGCParticle = JITParticle.add_variables( + [ + Variable("o2", dtype=np.float32, initial=np.nan), + Variable("chl", dtype=np.float32, initial=np.nan), + Variable("no3", dtype=np.float32, initial=np.nan), + Variable("po4", dtype=np.float32, initial=np.nan), + Variable("ph", dtype=np.float32, initial=np.nan), + Variable("phyc", dtype=np.float32, initial=np.nan), + Variable("zooc", dtype=np.float32, initial=np.nan), + Variable("nppv", dtype=np.float32, initial=np.nan), + Variable("raising", dtype=np.int8, initial=0.0), # bool. 0 is False, 1 is True. + Variable("max_depth", dtype=np.float32), + Variable("min_depth", dtype=np.float32), + Variable("winch_speed", dtype=np.float32), + ] +) + + +def _sample_o2(particle, fieldset, time): + particle.o2 = fieldset.o2[time, particle.depth, particle.lat, particle.lon] + + +def _sample_chlorophyll(particle, fieldset, time): + particle.chl = fieldset.chl[time, particle.depth, particle.lat, particle.lon] + + +def _sample_nitrate(particle, fieldset, time): + particle.no3 = fieldset.no3[time, particle.depth, particle.lat, particle.lon] + + +def _sample_phosphate(particle, fieldset, time): + particle.po4 = fieldset.po4[time, particle.depth, particle.lat, particle.lon] + + +def _sample_ph(particle, fieldset, time): + particle.ph = fieldset.ph[time, particle.depth, particle.lat, particle.lon] + + +def _sample_phytoplankton(particle, fieldset, time): + particle.phyc = fieldset.phyc[time, particle.depth, particle.lat, particle.lon] + + +def _sample_zooplankton(particle, fieldset, time): + particle.zooc = fieldset.zooc[time, particle.depth, particle.lat, particle.lon] + + +def _sample_primary_production(particle, fieldset, time): + particle.nppv = fieldset.nppv[time, particle.depth, particle.lat, particle.lon] + + +def _ctd_bgc_cast(particle, fieldset, time): + # lowering + if particle.raising == 0: + particle_ddepth = -particle.winch_speed * particle.dt + if particle.depth + particle_ddepth < particle.max_depth: + particle.raising = 1 + particle_ddepth = -particle_ddepth + # raising + else: + particle_ddepth = particle.winch_speed * particle.dt + if particle.depth + particle_ddepth > particle.min_depth: + particle.delete() class CTD_BGCInputDataset(instruments.InputDataset): @@ -40,7 +104,7 @@ def __init__(self, data_dir, credentials, space_time_region): space_time_region, ) - def datasets_dir(self) -> dict: + def get_datasets_dict(self) -> dict: """Variable specific args for instrument.""" return { "o2data": { @@ -86,18 +150,118 @@ def datasets_dir(self) -> dict: } -class CTD_BGCInstrument(instruments.Instrument): - """CTD_BGC instrument class.""" +# class CTD_BGCInstrument(instruments.Instrument): +# """CTD_BGC instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(CTD_BGC.name, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +def simulate_ctd_bgc( + fieldset: FieldSet, + out_path: str | Path, + ctd_bgcs: list[CTD_BGC], + outputdt: timedelta, +) -> None: + """ + Use Parcels to simulate a set of BGC CTDs in a fieldset. + + :param fieldset: The fieldset to simulate the BGC CTDs in. + :param out_path: The path to write the results to. + :param ctds: A list of BGC CTDs to simulate. + :param outputdt: Interval which dictates the update frequency of file output during simulation + :raises ValueError: Whenever provided BGC CTDs, fieldset, are not compatible with this function. + """ + WINCH_SPEED = 1.0 # sink and rise speed in m/s + DT = 10.0 # dt of CTD simulation integrator + + if len(ctd_bgcs) == 0: + print( + "No BGC CTDs provided. Parcels currently crashes when providing an empty particle set, so no BGC CTD simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - def __init__( - self, - config, - input_dataset, - kernels, + # deploy time for all ctds should be later than fieldset start time + if not all( + [ + np.datetime64(ctd_bgc.spacetime.time) >= fieldset_starttime + for ctd_bgc in ctd_bgcs + ] ): - """Initialise with instrument's name.""" - super().__init__(CTD_BGC.name, config, input_dataset, kernels) + raise ValueError("BGC CTD deployed before fieldset starts.") + + # depth the bgc ctd will go to. shallowest between bgc ctd max depth and bathymetry. + max_depths = [ + max( + ctd_bgc.max_depth, + fieldset.bathymetry.eval( + z=0, + y=ctd_bgc.spacetime.location.lat, + x=ctd_bgc.spacetime.location.lon, + time=0, + ), + ) + for ctd_bgc in ctd_bgcs + ] + + # CTD depth can not be too shallow, because kernel would break. + # This shallow is not useful anyway, no need to support. + if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): + raise ValueError( + f"BGC CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" + ) - def simulate(self): - """Simulate measurements.""" - ... + # define parcel particles + ctd_bgc_particleset = ParticleSet( + fieldset=fieldset, + pclass=_CTD_BGCParticle, + lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in ctd_bgcs], + lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in ctd_bgcs], + depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], + time=[ctd_bgc.spacetime.time for ctd_bgc in ctd_bgcs], + max_depth=max_depths, + min_depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], + winch_speed=[WINCH_SPEED for _ in ctd_bgcs], + ) + + # define output file for the simulation + out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=outputdt) + + # execute simulation + ctd_bgc_particleset.execute( + [ + _sample_o2, + _sample_chlorophyll, + _sample_nitrate, + _sample_phosphate, + _sample_ph, + _sample_phytoplankton, + _sample_zooplankton, + _sample_primary_production, + _ctd_bgc_cast, + ], + endtime=fieldset_endtime, + dt=DT, + verbose_progress=False, + output_file=out_file, + ) + + # there should be no particles left, as they delete themselves when they resurface + if len(ctd_bgc_particleset.particledata) != 0: + raise ValueError( + "Simulation ended before BGC CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." + ) diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 70ac4b7b..d1e24390 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -1,7 +1,11 @@ from dataclasses import dataclass -from datetime import timedelta +from datetime import datetime, timedelta +from pathlib import Path from typing import ClassVar +import numpy as np +from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable + from virtualship.models import Spacetime, instruments ## TODO: __init__.py will also need updating! @@ -18,9 +22,25 @@ class Drifter: lifetime: timedelta | None # if none, lifetime is infinite -# --------------- -# TODO: KERNELS -# --------------- +_DrifterParticle = JITParticle.add_variables( + [ + Variable("temperature", dtype=np.float32, initial=np.nan), + Variable("has_lifetime", dtype=np.int8), # bool + Variable("age", dtype=np.float32, initial=0.0), + Variable("lifetime", dtype=np.float32), + ] +) + + +def _sample_temperature(particle, fieldset, time): + particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] + + +def _check_lifetime(particle, fieldset, time): + if particle.has_lifetime == 1: + particle.age += particle.dt + if particle.age >= particle.lifetime: + particle.delete() class DrifterInputDataset(instruments.InputDataset): @@ -62,18 +82,91 @@ def get_datasets_dict(self) -> dict: } -class DrifterInstrument(instruments.Instrument): - """Drifter instrument class.""" - - def __init__( - self, - config, - input_dataset, - kernels, +# class DrifterInstrument(instruments.Instrument): +# """Drifter instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(Drifter.name, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +def simulate_drifters( + fieldset: FieldSet, + out_path: str | Path, + drifters: list[Drifter], + outputdt: timedelta, + dt: timedelta, + endtime: datetime | None = None, +) -> None: + """ + Use Parcels to simulate a set of drifters in a fieldset. + + :param fieldset: The fieldset to simulate the Drifters in. + :param out_path: The path to write the results to. + :param drifters: A list of drifters to simulate. + :param outputdt: Interval which dictates the update frequency of file output during simulation. + :param dt: Dt for integration. + :param endtime: Stop at this time, or if None, continue until the end of the fieldset or until all drifters ended. If this is earlier than the last drifter ended or later than the end of the fieldset, a warning will be printed. + """ + if len(drifters) == 0: + print( + "No drifters provided. Parcels currently crashes when providing an empty particle set, so no drifter simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + # define parcel particles + drifter_particleset = ParticleSet( + fieldset=fieldset, + pclass=_DrifterParticle, + lat=[drifter.spacetime.location.lat for drifter in drifters], + lon=[drifter.spacetime.location.lon for drifter in drifters], + depth=[drifter.depth for drifter in drifters], + time=[drifter.spacetime.time for drifter in drifters], + has_lifetime=[1 if drifter.lifetime is not None else 0 for drifter in drifters], + lifetime=[ + 0 if drifter.lifetime is None else drifter.lifetime.total_seconds() + for drifter in drifters + ], + ) + + # define output file for the simulation + out_file = drifter_particleset.ParticleFile( + name=out_path, outputdt=outputdt, chunks=[len(drifter_particleset), 100] + ) + + # get earliest between fieldset end time and provide end time + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + if endtime is None: + actual_endtime = fieldset_endtime + elif endtime > fieldset_endtime: + print("WARN: Requested end time later than fieldset end time.") + actual_endtime = fieldset_endtime + else: + actual_endtime = np.timedelta64(endtime) + + # execute simulation + drifter_particleset.execute( + [AdvectionRK4, _sample_temperature, _check_lifetime], + endtime=actual_endtime, + dt=dt, + output_file=out_file, + verbose_progress=True, + ) + + # if there are more particles left than the number of drifters with an indefinite endtime, warn the user + if len(drifter_particleset.particledata) > len( + [d for d in drifters if d.lifetime is None] ): - """Initialise with instrument's name.""" - super().__init__(Drifter.name, config, input_dataset, kernels) - - def simulate(self): - """Simulate measurements.""" - ... + print( + "WARN: Some drifters had a life time beyond the end time of the fieldset or the requested end time." + ) diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index f364b156..571a9ccd 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -1,7 +1,11 @@ from dataclasses import dataclass +from pathlib import Path from typing import ClassVar -from virtualship.models import instruments +import numpy as np +from parcels import FieldSet, ParticleSet, ScipyParticle, Variable + +from virtualship.models import Spacetime, instruments ## TODO: __init__.py will also need updating! # + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py @@ -14,9 +18,22 @@ class Underwater_ST: name: ClassVar[str] = "Underwater_ST" -# --------------- -# TODO: KERNELS -# --------------- +_ShipSTParticle = ScipyParticle.add_variables( + [ + Variable("S", dtype=np.float32, initial=np.nan), + Variable("T", dtype=np.float32, initial=np.nan), + ] +) + + +# define function sampling Salinity +def _sample_salinity(particle, fieldset, time): + particle.S = fieldset.S[time, particle.depth, particle.lat, particle.lon] + + +# define function sampling Temperature +def _sample_temperature(particle, fieldset, time): + particle.T = fieldset.T[time, particle.depth, particle.lat, particle.lon] class Underwater_STInputDataset(instruments.InputDataset): @@ -58,18 +75,67 @@ def get_datasets_dict(self) -> dict: } -class Underwater_STInstrument(instruments.Instrument): - """Underwater_ST instrument class.""" - - def __init__( - self, - config, - input_dataset, - kernels, - ): - """Initialise with instrument's name.""" - super().__init__(Underwater_ST.name, config, input_dataset, kernels) +# class Underwater_STInstrument(instruments.Instrument): +# """Underwater_ST instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(Underwater_ST.name, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +def simulate_ship_underwater_st( + fieldset: FieldSet, + out_path: str | Path, + depth: float, + sample_points: list[Spacetime], +) -> None: + """ + Use Parcels to simulate underway data, measuring salinity and temperature at the given depth along the ship track in a fieldset. + + :param fieldset: The fieldset to simulate the sampling in. + :param out_path: The path to write the results to. + :param depth: The depth at which to measure. 0 is water surface, negative is into the water. + :param sample_points: The places and times to sample at. + """ + sample_points.sort(key=lambda p: p.time) + + particleset = ParticleSet.from_list( + fieldset=fieldset, + pclass=_ShipSTParticle, + lon=0.0, # initial lat/lon are irrelevant and will be overruled later + lat=0.0, + depth=depth, + time=0, # same for time + ) + + # define output file for the simulation + # outputdt set to infinie as we want to just want to write at the end of every call to 'execute' + out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) + + # iterate over each point, manually set lat lon time, then + # execute the particle set for one step, performing one set of measurement + for point in sample_points: + particleset.lon_nextloop[:] = point.location.lon + particleset.lat_nextloop[:] = point.location.lat + particleset.time_nextloop[:] = fieldset.time_origin.reltime( + np.datetime64(point.time) + ) - def simulate(self): - """Simulate measurements.""" - ... + # perform one step using the particleset + # dt and runtime are set so exactly one step is made. + particleset.execute( + [_sample_salinity, _sample_temperature], + dt=1, + runtime=1, + verbose_progress=False, + output_file=out_file, + ) diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index dcec018b..4d90f3f1 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -1,7 +1,11 @@ from dataclasses import dataclass from datetime import timedelta +from pathlib import Path from typing import ClassVar +import numpy as np +from parcels import FieldSet, JITParticle, ParticleSet, Variable + from virtualship.models import Spacetime, instruments ## TODO: __init__.py will also need updating! @@ -18,9 +22,37 @@ class XBT: lifetime: timedelta | None # if none, lifetime is infinite -# --------------- -# TODO: KERNELS -# --------------- +_XBTParticle = JITParticle.add_variables( + [ + Variable("temperature", dtype=np.float32, initial=np.nan), + Variable("max_depth", dtype=np.float32), + Variable("min_depth", dtype=np.float32), + Variable("fall_speed", dtype=np.float32), + Variable("deceleration_coefficient", dtype=np.float32), + ] +) + + +def _sample_temperature(particle, fieldset, time): + particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] + + +def _xbt_cast(particle, fieldset, time): + particle_ddepth = -particle.fall_speed * particle.dt + + # update the fall speed from the quadractic fall-rate equation + # check https://doi.org/10.5194/os-7-231-2011 + particle.fall_speed = ( + particle.fall_speed - 2 * particle.deceleration_coefficient * particle.dt + ) + + # delete particle if depth is exactly max_depth + if particle.depth == particle.max_depth: + particle.delete() + + # set particle depth to max depth if it's too deep + if particle.depth + particle_ddepth < particle.max_depth: + particle_ddepth = particle.max_depth - particle.depth class XBTInputDataset(instruments.InputDataset): @@ -67,18 +99,105 @@ def get_datasets_dict(self) -> dict: } -class XBTInstrument(instruments.Instrument): - """XBT instrument class.""" +# class XBTInstrument(instruments.Instrument): +# """XBT instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(XBT.name, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +def simulate_xbt( + fieldset: FieldSet, + out_path: str | Path, + xbts: list[XBT], + outputdt: timedelta, +) -> None: + """ + Use Parcels to simulate a set of XBTs in a fieldset. + + :param fieldset: The fieldset to simulate the XBTs in. + :param out_path: The path to write the results to. + :param xbts: A list of XBTs to simulate. + :param outputdt: Interval which dictates the update frequency of file output during simulation + :raises ValueError: Whenever provided XBTs, fieldset, are not compatible with this function. + """ + DT = 10.0 # dt of XBT simulation integrator + + if len(xbts) == 0: + print( + "No XBTs provided. Parcels currently crashes when providing an empty particle set, so no XBT simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return - def __init__( - self, - config, - input_dataset, - kernels, - ): - """Initialise with instrument's name.""" - super().__init__(XBT.name, config, input_dataset, kernels) + fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - def simulate(self): - """Simulate measurements.""" - ... + # deploy time for all xbts should be later than fieldset start time + if not all( + [np.datetime64(xbt.spacetime.time) >= fieldset_starttime for xbt in xbts] + ): + raise ValueError("XBT deployed before fieldset starts.") + + # depth the xbt will go to. shallowest between xbt max depth and bathymetry. + max_depths = [ + max( + xbt.max_depth, + fieldset.bathymetry.eval( + z=0, y=xbt.spacetime.location.lat, x=xbt.spacetime.location.lon, time=0 + ), + ) + for xbt in xbts + ] + + # initial fall speeds + initial_fall_speeds = [xbt.fall_speed for xbt in xbts] + + # XBT depth can not be too shallow, because kernel would break. + # This shallow is not useful anyway, no need to support. + for max_depth, fall_speed in zip(max_depths, initial_fall_speeds, strict=False): + if not max_depth <= -DT * fall_speed: + raise ValueError( + f"XBT max_depth or bathymetry shallower than maximum {-DT * fall_speed}" + ) + + # define xbt particles + xbt_particleset = ParticleSet( + fieldset=fieldset, + pclass=_XBTParticle, + lon=[xbt.spacetime.location.lon for xbt in xbts], + lat=[xbt.spacetime.location.lat for xbt in xbts], + depth=[xbt.min_depth for xbt in xbts], + time=[xbt.spacetime.time for xbt in xbts], + max_depth=max_depths, + min_depth=[xbt.min_depth for xbt in xbts], + fall_speed=[xbt.fall_speed for xbt in xbts], + ) + + # define output file for the simulation + out_file = xbt_particleset.ParticleFile(name=out_path, outputdt=outputdt) + + # execute simulation + xbt_particleset.execute( + [_sample_temperature, _xbt_cast], + endtime=fieldset_endtime, + dt=DT, + verbose_progress=False, + output_file=out_file, + ) + + # there should be no particles left, as they delete themselves when they finish profiling + if len(xbt_particleset.particledata) != 0: + raise ValueError( + "Simulation ended before XBT finished profiling. This most likely means the field time dimension did not match the simulation time span." + ) From 8b5954d46b6075cd6b44b5bb4fb9570b4169ca7d Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 9 Oct 2025 13:57:59 +0200 Subject: [PATCH 11/97] improve/clarify comments and notes --- src/virtualship/cli/_fetch.py | 6 ++++-- src/virtualship/cli/_plan.py | 1 - src/virtualship/instruments/ctd.py | 3 --- src/virtualship/instruments/drifter.py | 3 --- src/virtualship/instruments/master.py | 2 +- .../instruments/ship_underwater_st.py | 3 --- src/virtualship/models/instruments.py | 16 ++++------------ 7 files changed, 9 insertions(+), 25 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index c258f044..2a9be43e 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -63,7 +63,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None expedition.schedule.space_time_region ) - # TODO: this (below) probably needs updating! + # TODO: needs updating? existing_download = get_existing_download(data_dir, space_time_region_hash) if existing_download is not None: click.echo( @@ -106,9 +106,11 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None ) shutil.copyfile(path / EXPEDITION, download_folder / EXPEDITION) + # TODO: enhance CLI output for users? + # bathymetry # TODO: this logic means it is downloaded for all expeditions but is only needed for CTD, CTD_BGC and XBT... - # TODO: to discuss: fine to still download for all expeditions because small size and then less duplication + # TODO: to discuss: fine to still download for all expeditions because small size and then less duplication? # TODO: or add as var in each of InputDataset objects per instrument because will be overwritten to disk anyway and therefore not duplicate? copernicusmarine.subset( dataset_id="cmems_mod_glo_phy_my_0.083deg_static", diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 87bfe336..1aa3208b 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -847,7 +847,6 @@ def compose(self) -> ComposeResult: yield Select( [ (str(year), year) - # TODO: change from hard coding? ...flexibility for different datasets... for year in range( 2022, datetime.datetime.now().year + 1, diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 070cdb78..b2f59ac5 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -8,9 +8,6 @@ from virtualship.models import Spacetime, instruments -## TODO: __init__.py will also need updating! -# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py - @dataclass class CTD: diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index d1e24390..8b0a1352 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -8,9 +8,6 @@ from virtualship.models import Spacetime, instruments -## TODO: __init__.py will also need updating! -# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py - @dataclass class Drifter: diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index 58eb0591..00f73891 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -6,7 +6,7 @@ class InstrumentType(Enum): # TODO: temporary measure so as not to have to overhaul the InstrumentType class logic in one go #! And also to avoid breaking other parts of the codebase which rely on InstrumentType when for now just working on fetch - # TODO: ideally this can evaporate in a future PR... + # TODO: ideally this can evaporate in the future... CTD = "CTD" CTD_BGC = "CTD_BGC" diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 571a9ccd..5fd39f2a 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -7,9 +7,6 @@ from virtualship.models import Spacetime, instruments -## TODO: __init__.py will also need updating! -# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py - @dataclass class Underwater_ST: diff --git a/src/virtualship/models/instruments.py b/src/virtualship/models/instruments.py index 7a356290..72d9237d 100644 --- a/src/virtualship/models/instruments.py +++ b/src/virtualship/models/instruments.py @@ -9,18 +9,10 @@ from virtualship.models.space_time_region import SpaceTimeRegion from virtualship.utils import ship_spinner -# TODO list START -# how much detail needs to be fed into InputDataset (i.e. how much it differs per instrument) -# may impact whether need a child class (e.g. CTDInputDataset) as well as just InputDataset -# or whether it could just be fed a `name` ... ? - -# ++ abc.abstractmethods could be useful for testing purposes...e.g. will fail if an instrumnet implementation doesn't adhere to the `Instrument` class standards - -# ++ discussion point with others, do we think it's okay to overhaul the data downloading so that each instrument has it's own files, rather than sharing data? -# ++ it's a cleaner way of making the whole repo more modular, i.e. have higher order logic for defining data downloads and housing all instrument logic in one place... -# ++ may even not matter so much considering working towards cloud integration... + we are not looking to optimise performance...? -# ++ OR, for now work on it in this way and then at the end make some clever changes to consolidate to minimum number of files dependent on instrument selections...? -# TODO list END +# TODO: +# Discussion: Should each instrument manage its own data files for modularity, +# or should we consolidate downloads to minimize file duplication across instruments? +# Consider starting with per-instrument files for simplicity, and refactor later if needed. class InputDataset(abc.ABC): From ff38f4f261a03e4018a35ce67020b70b3a27d259 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 9 Oct 2025 13:58:25 +0200 Subject: [PATCH 12/97] Refactor ArgoFloat and XBT classes to include depth parameters and remove outdated comments --- src/virtualship/instruments/argo_float.py | 11 ++++++----- src/virtualship/instruments/xbt.py | 10 +++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 4769eccf..a9f296d5 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -16,9 +16,6 @@ from virtualship.models import Spacetime, instruments -## TODO: __init__.py will also need updating! -# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py - @dataclass class ArgoFloat: @@ -26,8 +23,12 @@ class ArgoFloat: name: ClassVar[str] = "ArgoFloat" spacetime: Spacetime - depth: float # depth at which it floats and samples - lifetime: timedelta | None # if none, lifetime is infinite + min_depth: float + max_depth: float + drift_depth: float + vertical_speed: float + cycle_days: float + drift_days: float _ArgoParticle = JITParticle.add_variables( diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 4d90f3f1..ca831afc 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -8,9 +8,6 @@ from virtualship.models import Spacetime, instruments -## TODO: __init__.py will also need updating! -# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py - @dataclass class XBT: @@ -18,8 +15,10 @@ class XBT: name: ClassVar[str] = "XBT" spacetime: Spacetime - depth: float # depth at which it floats and samples - lifetime: timedelta | None # if none, lifetime is infinite + min_depth: float + max_depth: float + fall_speed: float + deceleration_coefficient: float _XBTParticle = JITParticle.add_variables( @@ -165,6 +164,7 @@ def simulate_xbt( # XBT depth can not be too shallow, because kernel would break. # This shallow is not useful anyway, no need to support. + # TODO: should this be more informative? Is "maximum" right? Should tell user can't use XBT here? for max_depth, fall_speed in zip(max_depths, initial_fall_speeds, strict=False): if not max_depth <= -DT * fall_speed: raise ValueError( From c870d1cbcf5c218cb9d8637acb45bca045c0435a Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 14 Oct 2025 11:59:25 +0200 Subject: [PATCH 13/97] avoid circular import issues --- src/virtualship/instruments/__init__.py | 2 ++ src/virtualship/instruments/adcp.py | 5 +++-- src/virtualship/instruments/argo_float.py | 5 +++-- src/virtualship/instruments/ctd.py | 5 +++-- src/virtualship/instruments/ctd_bgc.py | 5 +++-- src/virtualship/instruments/drifter.py | 5 +++-- src/virtualship/instruments/ship_underwater_st.py | 5 +++-- src/virtualship/instruments/xbt.py | 5 +++-- 8 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/virtualship/instruments/__init__.py b/src/virtualship/instruments/__init__.py index 6a6ffbca..a5da8dac 100644 --- a/src/virtualship/instruments/__init__.py +++ b/src/virtualship/instruments/__init__.py @@ -1,5 +1,7 @@ """Measurement instrument that can be used with Parcels.""" +from virtualship.models.spacetime import Spacetime # noqa: F401 + from . import adcp, argo_float, ctd, ctd_bgc, drifter, ship_underwater_st, xbt __all__ = [ diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 052004fb..ccf48738 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -5,7 +5,8 @@ import numpy as np from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.models import Spacetime, instruments +from virtualship.models.instruments import InputDataset +from virtualship.models.spacetime import Spacetime ## TODO: __init__.py will also need updating! # + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py @@ -34,7 +35,7 @@ def _sample_velocity(particle, fieldset, time): ) -class ADCPInputDataset(instruments.InputDataset): +class ADCPInputDataset(InputDataset): """Input dataset for ADCP instrument.""" DOWNLOAD_BUFFERS: ClassVar[dict] = { diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index a9f296d5..b9949fb1 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -14,7 +14,8 @@ Variable, ) -from virtualship.models import Spacetime, instruments +from virtualship.models.instruments import InputDataset +from virtualship.models.spacetime import Spacetime @dataclass @@ -115,7 +116,7 @@ def _check_error(particle, fieldset, time): particle.delete() -class ArgoFloatInputDataset(instruments.InputDataset): +class ArgoFloatInputDataset(InputDataset): """Input dataset for ArgoFloat instrument.""" DOWNLOAD_BUFFERS: ClassVar[dict] = { diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index b2f59ac5..15a46234 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -6,7 +6,8 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models import Spacetime, instruments +from virtualship.models.instruments import InputDataset +from virtualship.models.spacetime import Spacetime @dataclass @@ -53,7 +54,7 @@ def _ctd_cast(particle, fieldset, time): particle.delete() -class CTDInputDataset(instruments.InputDataset): +class CTDInputDataset(InputDataset): """Input dataset for CTD instrument.""" DOWNLOAD_BUFFERS: ClassVar[dict] = { diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 1025a5c8..fb9ea241 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -6,7 +6,8 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models import Spacetime, instruments +from virtualship.models.instruments import InputDataset +from virtualship.models.spacetime import Spacetime @dataclass @@ -83,7 +84,7 @@ def _ctd_bgc_cast(particle, fieldset, time): particle.delete() -class CTD_BGCInputDataset(instruments.InputDataset): +class CTD_BGCInputDataset(InputDataset): """Input dataset object for CTD_BGC instrument.""" DOWNLOAD_BUFFERS: ClassVar[dict] = { diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 8b0a1352..60ba5558 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -6,7 +6,8 @@ import numpy as np from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models import Spacetime, instruments +from virtualship.models.instruments import InputDataset +from virtualship.models.spacetime import Spacetime @dataclass @@ -40,7 +41,7 @@ def _check_lifetime(particle, fieldset, time): particle.delete() -class DrifterInputDataset(instruments.InputDataset): +class DrifterInputDataset(InputDataset): """Input dataset for Drifter instrument.""" DOWNLOAD_BUFFERS: ClassVar[dict] = { diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 5fd39f2a..332c0b2c 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -5,7 +5,8 @@ import numpy as np from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.models import Spacetime, instruments +from virtualship.models.instruments import InputDataset +from virtualship.models.spacetime import Spacetime @dataclass @@ -33,7 +34,7 @@ def _sample_temperature(particle, fieldset, time): particle.T = fieldset.T[time, particle.depth, particle.lat, particle.lon] -class Underwater_STInputDataset(instruments.InputDataset): +class Underwater_STInputDataset(InputDataset): """Input dataset for Underwater_ST instrument.""" DOWNLOAD_BUFFERS: ClassVar[dict] = { diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index ca831afc..81cba7b9 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -6,7 +6,8 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models import Spacetime, instruments +from virtualship.models.instruments import InputDataset +from virtualship.models.spacetime import Spacetime @dataclass @@ -54,7 +55,7 @@ def _xbt_cast(particle, fieldset, time): particle_ddepth = particle.max_depth - particle.depth -class XBTInputDataset(instruments.InputDataset): +class XBTInputDataset(InputDataset): """Input dataset for XBT instrument.""" DOWNLOAD_BUFFERS: ClassVar[dict] = { From 49b3beeb44f8938e16398828f8a1448384002d17 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 14 Oct 2025 12:01:27 +0200 Subject: [PATCH 14/97] make tests for InputDataset base class --- tests/models/test_instruments.py | 97 ++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 tests/models/test_instruments.py diff --git a/tests/models/test_instruments.py b/tests/models/test_instruments.py new file mode 100644 index 00000000..2ad73cdf --- /dev/null +++ b/tests/models/test_instruments.py @@ -0,0 +1,97 @@ +import datetime +from unittest.mock import patch + +import pytest + +from virtualship.models.instruments import InputDataset +from virtualship.models.space_time_region import ( + SpaceTimeRegion, + SpatialRange, + TimeRange, +) + + +class DummyInputDataset(InputDataset): + """A minimal InputDataset subclass for testing purposes.""" + + def get_datasets_dict(self): + """Return a dummy datasets dict for testing.""" + return { + "dummy": { + "dataset_id": "test_id", + "variables": ["var1"], + "output_filename": "dummy.nc", + } + } + + +@pytest.fixture +def dummy_space_time_region(): + spatial_range = SpatialRange( + minimum_longitude=0, + maximum_longitude=1, + minimum_latitude=0, + maximum_latitude=1, + minimum_depth=0, + maximum_depth=10, + ) + base_time = datetime.datetime.strptime("1950-01-01", "%Y-%m-%d") + time_range = TimeRange( + start_time=base_time, + end_time=base_time + datetime.timedelta(hours=1), + ) + return SpaceTimeRegion( + spatial_range=spatial_range, + time_range=time_range, + ) + + +def test_inputdataset_abstract_instantiation(): + # instantiation should not be allowed + with pytest.raises(TypeError): + InputDataset( + name="test", + latlon_buffer=0, + datetime_buffer=0, + min_depth=0, + max_depth=10, + data_dir=".", + credentials={"username": "u", "password": "p"}, + space_time_region=None, + ) + + +def test_dummyinputdataset_initialization(dummy_space_time_region): + ds = DummyInputDataset( + name="test", + latlon_buffer=0.5, + datetime_buffer=1, + min_depth=0, + max_depth=10, + data_dir=".", + credentials={"username": "u", "password": "p"}, + space_time_region=dummy_space_time_region, + ) + assert ds.name == "test" + assert ds.latlon_buffer == 0.5 + assert ds.datetime_buffer == 1 + assert ds.min_depth == 0 + assert ds.max_depth == 10 + assert ds.data_dir == "." + assert ds.credentials["username"] == "u" + + +@patch("virtualship.models.instruments.copernicusmarine.subset") +def test_download_data_calls_subset(mock_subset, dummy_space_time_region): + ds = DummyInputDataset( + name="test", + latlon_buffer=0.5, + datetime_buffer=1, + min_depth=0, + max_depth=10, + data_dir=".", + credentials={"username": "u", "password": "p"}, + space_time_region=dummy_space_time_region, + ) + ds.download_data() + assert mock_subset.called From c4e31963d8d23f71c6e051228ff0b70cff3f8c0d Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 22 Oct 2025 10:40:39 +0200 Subject: [PATCH 15/97] refactor instrument handling in _fetch.py and update expedition model to include get_instruments method --- src/virtualship/cli/_fetch.py | 149 ++---------------------- src/virtualship/instruments/__init__.py | 16 +-- src/virtualship/instruments/master.py | 4 +- src/virtualship/models/__init__.py | 2 - src/virtualship/models/expedition.py | 39 +++---- 5 files changed, 30 insertions(+), 180 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 2a9be43e..c460db37 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -11,6 +11,7 @@ from pydantic import BaseModel from virtualship.errors import IncompleteDownloadError +from virtualship.instruments.master import get_instruments_registry from virtualship.utils import ( _dump_yaml, _generic_load_yaml, @@ -24,8 +25,6 @@ import virtualship.cli._creds as creds from virtualship.utils import EXPEDITION -from virtualship.instruments.master import INSTRUMENTS -from virtualship.instruments.master import InstrumentType, get_instruments_registry DOWNLOAD_METADATA = "download_metadata.yaml" @@ -63,7 +62,6 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None expedition.schedule.space_time_region ) - # TODO: needs updating? existing_download = get_existing_download(data_dir, space_time_region_hash) if existing_download is not None: click.echo( @@ -77,26 +75,9 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None username, password, creds_path ) - # Extract space_time_region details from the schedule - spatial_range = expedition.schedule.space_time_region.spatial_range - time_range = expedition.schedule.space_time_region.time_range - start_datetime = time_range.start_time - end_datetime = time_range.end_time - instruments_in_schedule = expedition.schedule.get_instruments() - - # TEMPORARY measure to get underway instruments in `instruments_in_schedule` - # TODO: should evaporate when schedule and ship_config.yaml files are consolidated in a separate PR... - if ship_config.adcp_config is not None: - instruments_in_schedule.add(InstrumentType.ADCP) - if ship_config.ship_underwater_st_config is not None: - instruments_in_schedule.add(InstrumentType.UNDERWATER_ST) - - # TEMPORARY measure to get underway instruments in `instruments_in_schedule` - # TODO: should evaporate when schedule and ship_config.yaml files are consolidated in a separate PR... - if ship_config.adcp_config is not None: - instruments_in_schedule.add(InstrumentType.ADCP) - if ship_config.ship_underwater_st_config is not None: - instruments_in_schedule.add(InstrumentType.UNDERWATER_ST) + # Extract instruments and space_time_region details from expedition + instruments_in_expedition = expedition.get_instruments() + space_time_region = expedition.schedule.space_time_region # Create download folder and set download metadata download_folder = data_dir / hash_to_filename(space_time_region_hash) @@ -108,10 +89,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None # TODO: enhance CLI output for users? - # bathymetry - # TODO: this logic means it is downloaded for all expeditions but is only needed for CTD, CTD_BGC and XBT... - # TODO: to discuss: fine to still download for all expeditions because small size and then less duplication? - # TODO: or add as var in each of InputDataset objects per instrument because will be overwritten to disk anyway and therefore not duplicate? + # bathymetry (for all expeditions) copernicusmarine.subset( dataset_id="cmems_mod_glo_phy_my_0.083deg_static", variables=["deptho"], @@ -131,9 +109,9 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None coordinates_selection_method="outside", ) - # keep only instruments in INTSTRUMENTS which are in schedule + # access instrument classes but keep only instruments which are in schedule filter_instruments = { - k: v for k, v in INSTRUMENTS.items() if k in instruments_in_schedule + k: v for k, v in INSTRUMENTS.items() if k in instruments_in_expedition } # iterate across instruments and download data based on space_time_region @@ -142,116 +120,11 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None input_dataset = instrument["input_class"]( data_dir=download_folder, credentials=credentials, - space_time_region=space_time_region, + space_time_region=expedition.space_time_region, ) - #! - #### TODO - # ++ new logic here where iterates (?) through available instruments and determines whether download is required: - # ++ by conditions of: - # 1) whether it's in the schedule (and from this be able to call the right classes from the instruments directory?) and - #! 2) is there a clever way of not unnecessarily duplicating data downloads if instruments use the same?! - # (try with a version first where does them all in tow and then try and optimise...?) - - #! - ## TODO: move to generic bathymetry download which is done for all expeditions - - if ( - ( - {"XBT", "CTD", "CDT_BGC", "SHIP_UNDERWATER_ST"} - & set(instrument.name for instrument in instruments_in_schedule) - ) - or expedition.instruments_config.ship_underwater_st_config is not None - or expedition.instruments_config.adcp_config is not None - ): - print("Ship data will be downloaded. Please wait...") - - # Define all ship datasets to download, including bathymetry - download_dict = { - "Bathymetry": { - "dataset_id": "cmems_mod_glo_phy_my_0.083deg_static", - "variables": ["deptho"], - "output_filename": "bathymetry.nc", - }, - "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", - "variables": ["uo", "vo"], - "output_filename": "ship_uv.nc", - }, - "Sdata": { - "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", - "variables": ["so"], - "output_filename": "ship_s.nc", - }, - "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", - "variables": ["thetao"], - "output_filename": "ship_t.nc", - }, - } - - # Iterate over all datasets and download each based on space_time_region - try: - for dataset in download_dict.values(): - copernicusmarine.subset( - dataset_id=dataset["dataset_id"], - variables=dataset["variables"], - minimum_longitude=spatial_range.minimum_longitude, - maximum_longitude=spatial_range.maximum_longitude, - minimum_latitude=spatial_range.minimum_latitude, - maximum_latitude=spatial_range.maximum_latitude, - start_datetime=start_datetime, - end_datetime=end_datetime, - minimum_depth=abs(spatial_range.minimum_depth), - maximum_depth=abs(spatial_range.maximum_depth), - output_filename=dataset["output_filename"], - output_directory=download_folder, - username=username, - password=password, - overwrite=True, - coordinates_selection_method="outside", - ) - except InvalidUsernameOrPassword as e: - shutil.rmtree(download_folder) - raise e - click.echo("Ship data download based on space-time region completed.") - - if InstrumentType.DRIFTER in instruments_in_schedule: - print("Drifter data will be downloaded. Please wait...") - drifter_download_dict = { - "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", - "variables": ["uo", "vo"], - "output_filename": "drifter_uv.nc", - }, - "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", - "variables": ["thetao"], - "output_filename": "drifter_t.nc", - }, - } - - # Iterate over all datasets and download each based on space_time_region - try: - for dataset in drifter_download_dict.values(): - copernicusmarine.subset( - dataset_id=dataset["dataset_id"], - variables=dataset["variables"], - minimum_longitude=spatial_range.minimum_longitude - 3.0, - maximum_longitude=spatial_range.maximum_longitude + 3.0, - minimum_latitude=spatial_range.minimum_latitude - 3.0, - maximum_latitude=spatial_range.maximum_latitude + 3.0, - start_datetime=start_datetime, - end_datetime=end_datetime + timedelta(days=21), - minimum_depth=abs(1), - maximum_depth=abs(1), - output_filename=dataset["output_filename"], - output_directory=download_folder, - username=username, - password=password, - overwrite=True, - coordinates_selection_method="outside", - ) + input_dataset.download_data() + except InvalidUsernameOrPassword as e: shutil.rmtree(download_folder) raise e @@ -331,11 +204,9 @@ def get_existing_download(data_dir: Path, space_time_region_hash: str) -> Path | hash = filename_to_hash(download_path.name) except ValueError: continue - if hash == space_time_region_hash: assert_complete_download(download_path) return download_path - return None diff --git a/src/virtualship/instruments/__init__.py b/src/virtualship/instruments/__init__.py index a5da8dac..5324da2c 100644 --- a/src/virtualship/instruments/__init__.py +++ b/src/virtualship/instruments/__init__.py @@ -1,15 +1 @@ -"""Measurement instrument that can be used with Parcels.""" - -from virtualship.models.spacetime import Spacetime # noqa: F401 - -from . import adcp, argo_float, ctd, ctd_bgc, drifter, ship_underwater_st, xbt - -__all__ = [ - "adcp", - "argo_float", - "ctd", - "ctd_bgc", - "drifter", - "ship_underwater_st", - "xbt", -] +"""Instruments in VirtualShip.""" diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index 00f73891..cb31468f 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -4,9 +4,7 @@ class InstrumentType(Enum): """Types of the instruments.""" - # TODO: temporary measure so as not to have to overhaul the InstrumentType class logic in one go - #! And also to avoid breaking other parts of the codebase which rely on InstrumentType when for now just working on fetch - # TODO: ideally this can evaporate in the future... + # TODO: scope for this to evaporate in the future...? CTD = "CTD" CTD_BGC = "CTD_BGC" diff --git a/src/virtualship/models/__init__.py b/src/virtualship/models/__init__.py index a2f1546c..5eaabb85 100644 --- a/src/virtualship/models/__init__.py +++ b/src/virtualship/models/__init__.py @@ -8,7 +8,6 @@ DrifterConfig, Expedition, InstrumentsConfig, - InstrumentType, Schedule, ShipConfig, ShipUnderwaterSTConfig, @@ -30,7 +29,6 @@ "Schedule", "ShipConfig", "Waypoint", - "InstrumentType", "ArgoFloatConfig", "ADCPConfig", "CTDConfig", diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 2e073b84..ef860625 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -2,7 +2,6 @@ import itertools from datetime import datetime, timedelta -from enum import Enum from typing import TYPE_CHECKING import pydantic @@ -10,6 +9,7 @@ import yaml from virtualship.errors import ConfigError, ScheduleError +from virtualship.instruments.master import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta from .location import Location @@ -45,6 +45,23 @@ def from_yaml(cls, file_path: str) -> Expedition: data = yaml.safe_load(file) return Expedition(**data) + def get_instruments(self) -> set[InstrumentType]: + """Return a set of unique InstrumentType enums used in the expedition.""" + instruments_in_expedition = [] + # from waypoints + for waypoint in self.schedule.waypoints: + if waypoint.instrument: + for instrument in waypoint.instrument: + if instrument: + instruments_in_expedition.append(instrument) + # check for underway instruments and add if present in expeditions + if self.instruments_config.adcp_config is not None: + instruments_in_expedition.append(InstrumentType.ADCP) + if self.instruments_config.ship_underwater_st_config is not None: + instruments_in_expedition.append(InstrumentType.UNDERWATER_ST) + + return set(instruments_in_expedition) + class ShipConfig(pydantic.BaseModel): """Configuration of the ship.""" @@ -64,16 +81,6 @@ class Schedule(pydantic.BaseModel): model_config = pydantic.ConfigDict(extra="forbid") - def get_instruments(self) -> set[InstrumentType]: - """Return a set of unique InstrumentType enums used in the schedule.""" - instruments_in_schedule = [] - for waypoint in self.waypoints: - if waypoint.instrument: - for instrument in waypoint.instrument: - if instrument: - instruments_in_schedule.append(instrument) - return set(instruments_in_schedule) - def verify( self, ship_speed: float, @@ -213,16 +220,6 @@ def serialize_instrument(self, instrument): return instrument.value if instrument else None -class InstrumentType(Enum): - """Types of the instruments.""" - - CTD = "CTD" - CTD_BGC = "CTD_BGC" - DRIFTER = "DRIFTER" - ARGO_FLOAT = "ARGO_FLOAT" - XBT = "XBT" - - class ArgoFloatConfig(pydantic.BaseModel): """Configuration for argos floats.""" From 2f40f7dff5755e5e173800e8ee81a95ab5640fdf Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 22 Oct 2025 15:21:02 +0200 Subject: [PATCH 16/97] refactor instrument error handling in Expedition model and remove Schedule and ShipConfig classes --- src/virtualship/models/expedition.py | 37 +-- src/virtualship/models/schedule.py | 236 -------------------- src/virtualship/models/ship_config.py | 310 -------------------------- 3 files changed, 24 insertions(+), 559 deletions(-) delete mode 100644 src/virtualship/models/schedule.py delete mode 100644 src/virtualship/models/ship_config.py diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index ef860625..7e206e8f 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -8,7 +8,7 @@ import pyproj import yaml -from virtualship.errors import ConfigError, ScheduleError +from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.instruments.master import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta @@ -54,13 +54,18 @@ def get_instruments(self) -> set[InstrumentType]: for instrument in waypoint.instrument: if instrument: instruments_in_expedition.append(instrument) - # check for underway instruments and add if present in expeditions - if self.instruments_config.adcp_config is not None: - instruments_in_expedition.append(InstrumentType.ADCP) - if self.instruments_config.ship_underwater_st_config is not None: - instruments_in_expedition.append(InstrumentType.UNDERWATER_ST) - return set(instruments_in_expedition) + # check for underway instruments and add if present in expeditions + try: + if self.instruments_config.adcp_config is not None: + instruments_in_expedition.append(InstrumentType.ADCP) + if self.instruments_config.ship_underwater_st_config is not None: + instruments_in_expedition.append(InstrumentType.UNDERWATER_ST) + return set(instruments_in_expedition) + except Exception as e: + raise InstrumentsConfigError( + "Underway instrument config attribute(s) are missing from YAML. Must be Config object or None." + ) from e class ShipConfig(pydantic.BaseModel): @@ -348,6 +353,7 @@ class XBTConfig(pydantic.BaseModel): class InstrumentsConfig(pydantic.BaseModel): + # TODO: refactor potential for this? Move explicit instrument_config's away from models/ dir? """Configuration of instruments.""" argo_float_config: ArgoFloatConfig | None = None @@ -401,38 +407,43 @@ class InstrumentsConfig(pydantic.BaseModel): model_config = pydantic.ConfigDict(extra="forbid") - def verify(self, schedule: Schedule) -> None: + def verify(self, expedition: Expedition) -> None: """ Verify instrument configurations against the schedule. Removes instrument configs not present in the schedule and checks that all scheduled instruments are configured. Raises ConfigError if any scheduled instrument is missing a config. """ - instruments_in_schedule = schedule.get_instruments() + instruments_in_expedition = expedition.get_instruments() instrument_config_map = { InstrumentType.ARGO_FLOAT: "argo_float_config", InstrumentType.DRIFTER: "drifter_config", InstrumentType.XBT: "xbt_config", InstrumentType.CTD: "ctd_config", InstrumentType.CTD_BGC: "ctd_bgc_config", + InstrumentType.ADCP: "adcp_config", + InstrumentType.UNDERWATER_ST: "ship_underwater_st_config", } # Remove configs for unused instruments for inst_type, config_attr in instrument_config_map.items(): - if hasattr(self, config_attr) and inst_type not in instruments_in_schedule: + if ( + hasattr(self, config_attr) + and inst_type not in instruments_in_expedition + ): print( f"{inst_type.value} configuration provided but not in schedule. Removing config." ) setattr(self, config_attr, None) # Check all scheduled instruments are configured - for inst_type in instruments_in_schedule: + for inst_type in instruments_in_expedition: config_attr = instrument_config_map.get(inst_type) if ( not config_attr or not hasattr(self, config_attr) or getattr(self, config_attr) is None ): - raise ConfigError( - f"Schedule includes instrument '{inst_type.value}', but instruments_config does not provide configuration for it." + raise InstrumentsConfigError( + f"Expedition includes instrument '{inst_type.value}', but instruments_config does not provide configuration for it." ) diff --git a/src/virtualship/models/schedule.py b/src/virtualship/models/schedule.py deleted file mode 100644 index f3e5dabe..00000000 --- a/src/virtualship/models/schedule.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Schedule class.""" - -from __future__ import annotations - -import itertools -from datetime import datetime, timedelta -from pathlib import Path -from typing import TYPE_CHECKING - -import pydantic -import pyproj -import yaml - -from virtualship.errors import ScheduleError -from virtualship.instruments.master import InstrumentType - -from .location import Location -from .space_time_region import SpaceTimeRegion - -if TYPE_CHECKING: - from parcels import FieldSet - - from virtualship.expedition.input_data import InputData - -projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") - - -class Waypoint(pydantic.BaseModel): - """A Waypoint to sail to with an optional time and an optional instrument.""" - - location: Location - time: datetime | None = None - instrument: InstrumentType | list[InstrumentType] | None = None - - @pydantic.field_serializer("instrument") - def serialize_instrument(self, instrument): - """Ensure InstrumentType is serialized as a string (or list of strings).""" - if isinstance(instrument, list): - return [inst.value for inst in instrument] - return instrument.value if instrument else None - - -class Schedule(pydantic.BaseModel): - """Schedule of the virtual ship.""" - - waypoints: list[Waypoint] - space_time_region: SpaceTimeRegion | None = None - - model_config = pydantic.ConfigDict(extra="forbid") - - def to_yaml(self, file_path: str | Path) -> None: - """ - Write schedule to yaml file. - - :param file_path: Path to the file to write to. - """ - with open(file_path, "w") as file: - yaml.dump( - self.model_dump( - by_alias=True, - ), - file, - ) - - @classmethod - def from_yaml(cls, file_path: str | Path) -> Schedule: - """ - Load schedule from yaml file. - - :param file_path: Path to the file to load from. - :returns: The schedule. - """ - with open(file_path) as file: - data = yaml.safe_load(file) - return Schedule(**data) - - def get_instruments(self) -> set[InstrumentType]: - """ - Retrieve a set of unique instruments used in the schedule. - - This method iterates through all waypoints in the schedule and collects - the instruments associated with each waypoint. It returns a set of unique - instruments, either as objects or as names. - - :raises CheckpointError: If the past waypoints in the given schedule - have been changed compared to the checkpoint. - :return: set: A set of unique instruments used in the schedule. - - """ - instruments_in_schedule = [] - for waypoint in self.waypoints: - if waypoint.instrument: - for instrument in waypoint.instrument: - if instrument: - instruments_in_schedule.append(instrument) - return set(instruments_in_schedule) - - def verify( - self, - ship_speed: float, - input_data: InputData | None, - *, - check_space_time_region: bool = False, - ignore_missing_fieldsets: bool = False, - ) -> None: - """ - Verify the feasibility and correctness of the schedule's waypoints. - - This method checks various conditions to ensure the schedule is valid: - 1. At least one waypoint is provided. - 2. The first waypoint has a specified time. - 3. Waypoint times are in ascending order. - 4. All waypoints are in water (not on land). - 5. The ship can arrive on time at each waypoint given its speed. - - :param ship_speed: The ship's speed in knots. - :param input_data: An InputData object containing fieldsets used to check if waypoints are on water. - :param check_space_time_region: whether to check for missing space_time_region. - :param ignore_missing_fieldsets: whether to ignore warning for missing field sets. - :raises PlanningError: If any of the verification checks fail, indicating infeasible or incorrect waypoints. - :raises NotImplementedError: If an instrument in the schedule is not implemented. - :return: None. The method doesn't return a value but raises exceptions if verification fails. - """ - print("\nVerifying route... ") - - if check_space_time_region and self.space_time_region is None: - raise ScheduleError( - "space_time_region not found in schedule, please define it to fetch the data." - ) - - if len(self.waypoints) == 0: - raise ScheduleError("At least one waypoint must be provided.") - - # check first waypoint has a time - if self.waypoints[0].time is None: - raise ScheduleError("First waypoint must have a specified time.") - - # check waypoint times are in ascending order - timed_waypoints = [wp for wp in self.waypoints if wp.time is not None] - checks = [ - next.time >= cur.time for cur, next in itertools.pairwise(timed_waypoints) - ] - if not all(checks): - invalid_i = [i for i, c in enumerate(checks) if c] - raise ScheduleError( - f"Waypoint(s) {', '.join(f'#{i + 1}' for i in invalid_i)}: each waypoint should be timed after all previous waypoints", - ) - - # check if all waypoints are in water - # this is done by picking an arbitrary provided fieldset and checking if UV is not zero - - # get all available fieldsets - available_fieldsets = [] - if input_data is not None: - fieldsets = [ - input_data.adcp_fieldset, - input_data.argo_float_fieldset, - input_data.ctd_fieldset, - input_data.drifter_fieldset, - input_data.ship_underwater_st_fieldset, - ] - for fs in fieldsets: - if fs is not None: - available_fieldsets.append(fs) - - # check if there are any fieldsets, else it's an error - if len(available_fieldsets) == 0: - if not ignore_missing_fieldsets: - print( - "Cannot verify because no fieldsets have been loaded. This is probably " - "because you are not using any instruments in your schedule. This is not a problem, " - "but carefully check your waypoint locations manually." - ) - - else: - # pick any - fieldset = available_fieldsets[0] - # get waypoints with 0 UV - land_waypoints = [ - (wp_i, wp) - for wp_i, wp in enumerate(self.waypoints) - if _is_on_land_zero_uv(fieldset, wp) - ] - # raise an error if there are any - if len(land_waypoints) > 0: - raise ScheduleError( - f"The following waypoints are on land: {['#' + str(wp_i) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" - ) - - # check that ship will arrive on time at each waypoint (in case no unexpected event happen) - time = self.waypoints[0].time - for wp_i, (wp, wp_next) in enumerate( - zip(self.waypoints, self.waypoints[1:], strict=False) - ): - if wp.instrument is InstrumentType.CTD: - time += timedelta(minutes=20) - - geodinv: tuple[float, float, float] = projection.inv( - wp.location.lon, - wp.location.lat, - wp_next.location.lon, - wp_next.location.lat, - ) - distance = geodinv[2] - - time_to_reach = timedelta(seconds=distance / ship_speed * 3600 / 1852) - arrival_time = time + time_to_reach - - if wp_next.time is None: - time = arrival_time - elif arrival_time > wp_next.time: - raise ScheduleError( - f"Waypoint planning is not valid: would arrive too late at waypoint number {wp_i + 2}. " - f"location: {wp_next.location} time: {wp_next.time} instrument: {wp_next.instrument}" - ) - else: - time = wp_next.time - - print("... All good to go!") - - -def _is_on_land_zero_uv(fieldset: FieldSet, waypoint: Waypoint) -> bool: - """ - Check if waypoint is on land by assuming zero velocity means land. - - :param fieldset: The fieldset to sample the velocity from. - :param waypoint: The waypoint to check. - :returns: If the waypoint is on land. - """ - return fieldset.UV.eval( - 0, - fieldset.gridset.grids[0].depth[0], - waypoint.location.lat, - waypoint.location.lon, - applyConversion=False, - ) == (0.0, 0.0) diff --git a/src/virtualship/models/ship_config.py b/src/virtualship/models/ship_config.py deleted file mode 100644 index ba7d221f..00000000 --- a/src/virtualship/models/ship_config.py +++ /dev/null @@ -1,310 +0,0 @@ -"""ShipConfig and supporting classes.""" - -from __future__ import annotations - -from datetime import timedelta -from pathlib import Path -from typing import TYPE_CHECKING - -import pydantic -import yaml - -from virtualship.errors import ConfigError -from virtualship.instruments.master import InstrumentType -from virtualship.utils import _validate_numeric_mins_to_timedelta - -if TYPE_CHECKING: - from .schedule import Schedule - - -class ArgoFloatConfig(pydantic.BaseModel): - """Configuration for argos floats.""" - - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - drift_depth_meter: float = pydantic.Field(le=0.0) - vertical_speed_meter_per_second: float = pydantic.Field(lt=0.0) - cycle_days: float = pydantic.Field(gt=0.0) - drift_days: float = pydantic.Field(gt=0.0) - - -class ADCPConfig(pydantic.BaseModel): - """Configuration for ADCP instrument.""" - - max_depth_meter: float = pydantic.Field(le=0.0) - num_bins: int = pydantic.Field(gt=0.0) - period: timedelta = pydantic.Field( - serialization_alias="period_minutes", - validation_alias="period_minutes", - gt=timedelta(), - ) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("period") - def _serialize_period(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("period", mode="before") - def _validate_period(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class CTDConfig(pydantic.BaseModel): - """Configuration for CTD instrument.""" - - stationkeeping_time: timedelta = pydantic.Field( - serialization_alias="stationkeeping_time_minutes", - validation_alias="stationkeeping_time_minutes", - gt=timedelta(), - ) - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("stationkeeping_time") - def _serialize_stationkeeping_time(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("stationkeeping_time", mode="before") - def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class CTD_BGCConfig(pydantic.BaseModel): - """Configuration for CTD_BGC instrument.""" - - stationkeeping_time: timedelta = pydantic.Field( - serialization_alias="stationkeeping_time_minutes", - validation_alias="stationkeeping_time_minutes", - gt=timedelta(), - ) - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("stationkeeping_time") - def _serialize_stationkeeping_time(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("stationkeeping_time", mode="before") - def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class ShipUnderwaterSTConfig(pydantic.BaseModel): - """Configuration for underwater ST.""" - - period: timedelta = pydantic.Field( - serialization_alias="period_minutes", - validation_alias="period_minutes", - gt=timedelta(), - ) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("period") - def _serialize_period(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("period", mode="before") - def _validate_period(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class DrifterConfig(pydantic.BaseModel): - """Configuration for drifters.""" - - depth_meter: float = pydantic.Field(le=0.0) - lifetime: timedelta = pydantic.Field( - serialization_alias="lifetime_minutes", - validation_alias="lifetime_minutes", - gt=timedelta(), - ) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("lifetime") - def _serialize_lifetime(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("lifetime", mode="before") - def _validate_lifetime(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class XBTConfig(pydantic.BaseModel): - """Configuration for xbt instrument.""" - - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - fall_speed_meter_per_second: float = pydantic.Field(gt=0.0) - deceleration_coefficient: float = pydantic.Field(gt=0.0) - - -class ShipConfig(pydantic.BaseModel): - """Configuration of the virtual ship.""" - - ship_speed_knots: float = pydantic.Field(gt=0.0) - """ - Velocity of the ship in knots. - """ - - argo_float_config: ArgoFloatConfig | None = None - """ - Argo float configuration. - - If None, no argo floats can be deployed. - """ - - adcp_config: ADCPConfig | None = None - """ - ADCP configuration. - - If None, no ADCP measurements will be performed. - """ - - ctd_config: CTDConfig | None = None - """ - CTD configuration. - - If None, no CTDs can be cast. - """ - - ctd_bgc_config: CTD_BGCConfig | None = None - """ - CTD_BGC configuration. - - If None, no BGC CTDs can be cast. - """ - - ship_underwater_st_config: ShipUnderwaterSTConfig | None = None - """ - Ship underwater salinity temperature measurementconfiguration. - - If None, no ST measurements will be performed. - """ - - drifter_config: DrifterConfig | None = None - """ - Drifter configuration. - - If None, no drifters can be deployed. - """ - - xbt_config: XBTConfig | None = None - """ - XBT configuration. - - If None, no XBTs can be cast. - """ - - model_config = pydantic.ConfigDict(extra="forbid") - - def to_yaml(self, file_path: str | Path) -> None: - """ - Write config to yaml file. - - :param file_path: Path to the file to write to. - """ - with open(file_path, "w") as file: - yaml.dump(self.model_dump(by_alias=True), file) - - @classmethod - def from_yaml(cls, file_path: str | Path) -> ShipConfig: - """ - Load config from yaml file. - - :param file_path: Path to the file to load from. - :returns: The config. - """ - with open(file_path) as file: - data = yaml.safe_load(file) - return ShipConfig(**data) - - def verify(self, schedule: Schedule) -> None: - """ - Verify the ship configuration against the provided schedule. - - This function performs two main tasks: - 1. Removes instrument configurations that are not present in the schedule. - 2. Verifies that all instruments in the schedule have corresponding configurations. - - Parameters - ---------- - schedule : Schedule - The schedule object containing the planned instruments and waypoints. - - Returns - ------- - None - - Raises - ------ - ConfigError - If an instrument in the schedule does not have a corresponding configuration. - - Notes - ----- - - Prints a message if a configuration is provided for an instrument not in the schedule. - - Sets the configuration to None for instruments not in the schedule. - - Raises a ConfigError for each instrument in the schedule that lacks a configuration. - - """ - instruments_in_schedule = schedule.get_instruments() - - for instrument in [ - "ARGO_FLOAT", - "DRIFTER", - "XBT", - "CTD", - "CTD_BGC", - ]: # TODO make instrument names consistent capitals or lowercase throughout codebase - if hasattr(self, instrument.lower() + "_config") and not any( - instrument == schedule_instrument.name - for schedule_instrument in instruments_in_schedule - ): - print(f"{instrument} configuration provided but not in schedule.") - setattr(self, instrument.lower() + "_config", None) - - # verify instruments in schedule have configuration - # TODO: the ConfigError message could be improved to explain that the **schedule** file has X instrument but the **ship_config** file does not - for instrument in instruments_in_schedule: - try: - InstrumentType(instrument) - except ValueError as e: - raise NotImplementedError("Instrument not supported.") from e - - if instrument == InstrumentType.ARGO_FLOAT and ( - not hasattr(self, "argo_float_config") or self.argo_float_config is None - ): - raise ConfigError( - "Planning has a waypoint with Argo float instrument, but configuration does not configure Argo floats." - ) - if instrument == InstrumentType.CTD and ( - not hasattr(self, "ctd_config") or self.ctd_config is None - ): - raise ConfigError( - "Planning has a waypoint with CTD instrument, but configuration does not configure CTDs." - ) - if instrument == InstrumentType.CTD_BGC and ( - not hasattr(self, "ctd_bgc_config") or self.ctd_bgc_config is None - ): - raise ConfigError( - "Planning has a waypoint with CTD_BGC instrument, but configuration does not configure CTD_BGCs." - ) - if instrument == InstrumentType.DRIFTER and ( - not hasattr(self, "drifter_config") or self.drifter_config is None - ): - raise ConfigError( - "Planning has a waypoint with drifter instrument, but configuration does not configure drifters." - ) - - if instrument == InstrumentType.XBT and ( - not hasattr(self, "xbt_config") or self.xbt_config is None - ): - raise ConfigError( - "Planning has a waypoint with XBT instrument, but configuration does not configure XBT." - ) From 43c855d9536ef648ce07fa86d22077aad1870fe1 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 22 Oct 2025 15:23:59 +0200 Subject: [PATCH 17/97] add is_underway property to InstrumentType and filter instruments in plan UI --- src/virtualship/cli/_plan.py | 6 +++--- src/virtualship/instruments/master.py | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 1aa3208b..435d5ab0 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -29,6 +29,7 @@ type_to_textual, ) from virtualship.errors import UnexpectedError, UserError +from virtualship.instruments.master import InstrumentType from virtualship.models import ( ADCPConfig, ArgoFloatConfig, @@ -36,7 +37,6 @@ CTDConfig, DrifterConfig, Expedition, - InstrumentType, Location, ShipConfig, ShipUnderwaterSTConfig, @@ -630,7 +630,7 @@ def _update_schedule(self): 0, ) wp.instrument = [] - for instrument in InstrumentType: + for instrument in [inst for inst in InstrumentType if not inst.is_underway]: switch_on = self.query_one(f"#wp{i}_{instrument.value}").value if instrument.value == "DRIFTER" and switch_on: count_str = self.query_one(f"#wp{i}_drifter_count").value @@ -901,7 +901,7 @@ def compose(self) -> ComposeResult: ) yield Label("Instruments:") - for instrument in InstrumentType: + for instrument in [i for i in InstrumentType if not i.is_underway]: is_selected = instrument in (self.waypoint.instrument or []) with Horizontal(): yield Label(instrument.value) diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index cb31468f..6e098988 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -14,6 +14,11 @@ class InstrumentType(Enum): ADCP = "ADCP" UNDERWATER_ST = "UNDERWATER_ST" + @property + def is_underway(self) -> bool: + """Return True if instrument is an underway instrument (ADCP, UNDERWATER_ST).""" + return self in {InstrumentType.ADCP, InstrumentType.UNDERWATER_ST} + def get_instruments_registry(): # local imports to avoid circular import issues From 79c81cb69ea8e25c669f8b4e9a58f16e95572504 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 22 Oct 2025 15:24:59 +0200 Subject: [PATCH 18/97] enhance CLI output for fetching --- src/virtualship/cli/_fetch.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index c460db37..1cf33fa9 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -87,7 +87,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None ) shutil.copyfile(path / EXPEDITION, download_folder / EXPEDITION) - # TODO: enhance CLI output for users? + click.echo(f"\n\n{(' Fetching data for: Bathymetry ').center(80, '=')}\n\n") # bathymetry (for all expeditions) copernicusmarine.subset( @@ -116,11 +116,14 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None # iterate across instruments and download data based on space_time_region for itype, instrument in filter_instruments.items(): + click.echo( + f"\n\n{(' Fetching data for: ' + itype.value + ' ').center(80, '=')}\n\n" + ) try: input_dataset = instrument["input_class"]( data_dir=download_folder, credentials=credentials, - space_time_region=expedition.space_time_region, + space_time_region=space_time_region, ) input_dataset.download_data() From efb53cada0ac63e87f8018aa1e8c0353a41039f7 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 22 Oct 2025 15:25:52 +0200 Subject: [PATCH 19/97] general fixes and new error class --- src/virtualship/errors.py | 8 +++++++- src/virtualship/expedition/checkpoint.py | 3 ++- src/virtualship/expedition/do_expedition.py | 2 +- src/virtualship/expedition/simulate_schedule.py | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/virtualship/errors.py b/src/virtualship/errors.py index cdd58349..3ba52a9a 100644 --- a/src/virtualship/errors.py +++ b/src/virtualship/errors.py @@ -22,7 +22,7 @@ class ScheduleError(RuntimeError): pass -class ConfigError(RuntimeError): +class InstrumentsConfigError(RuntimeError): """An error in the config.""" pass @@ -38,3 +38,9 @@ class UnexpectedError(Exception): """Error raised when there is an unexpected problem.""" pass + + +class UnderwayConfigsError(Exception): + """Error raised when underway instrument configurations (ADCP or underwater ST) are missing.""" + + pass diff --git a/src/virtualship/expedition/checkpoint.py b/src/virtualship/expedition/checkpoint.py index 6daf1a9b..ff2dadd6 100644 --- a/src/virtualship/expedition/checkpoint.py +++ b/src/virtualship/expedition/checkpoint.py @@ -8,7 +8,8 @@ import yaml from virtualship.errors import CheckpointError -from virtualship.models import InstrumentType, Schedule +from virtualship.instruments.master import InstrumentType +from virtualship.models import Schedule class _YamlDumper(yaml.SafeDumper): diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index 5c46d2eb..921ea528 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -40,7 +40,7 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> expedition = _get_expedition(expedition_dir) # Verify instruments_config file is consistent with schedule - expedition.instruments_config.verify(expedition.schedule) + expedition.instruments_config.verify(expedition) # load last checkpoint checkpoint = _load_checkpoint(expedition_dir) diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 3b78c5c7..784e2d32 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -11,10 +11,10 @@ from virtualship.instruments.ctd import CTD from virtualship.instruments.ctd_bgc import CTD_BGC from virtualship.instruments.drifter import Drifter +from virtualship.instruments.master import InstrumentType from virtualship.instruments.xbt import XBT from virtualship.models import ( Expedition, - InstrumentType, Location, Spacetime, Waypoint, From c4ddea1aeadef5ecba43abfe812eae744aa8518a Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 22 Oct 2025 15:26:16 +0200 Subject: [PATCH 20/97] refactor test cases to use Expedition object --- tests/expedition/test_expedition.py | 140 +++++++++++++++++----------- 1 file changed, 85 insertions(+), 55 deletions(-) diff --git a/tests/expedition/test_expedition.py b/tests/expedition/test_expedition.py index a4643e03..4bed35e7 100644 --- a/tests/expedition/test_expedition.py +++ b/tests/expedition/test_expedition.py @@ -4,9 +4,14 @@ import pyproj import pytest -from virtualship.errors import ConfigError, ScheduleError +from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.expedition.do_expedition import _load_input_data -from virtualship.models import Expedition, Location, Schedule, Waypoint +from virtualship.models import ( + Expedition, + Location, + Schedule, + Waypoint, +) from virtualship.utils import EXPEDITION, _get_expedition, get_example_expedition projection = pyproj.Geod(ellps="WGS84") @@ -56,6 +61,7 @@ def test_verify_schedule() -> None: def test_get_instruments() -> None: + get_expedition = _get_expedition(expedition_dir) schedule = Schedule( waypoints=[ Waypoint(location=Location(0, 0), instrument=["CTD"]), @@ -63,12 +69,21 @@ def test_get_instruments() -> None: Waypoint(location=Location(1, 0), instrument=["CTD"]), ] ) - - assert set(instrument.name for instrument in schedule.get_instruments()) == { - "CTD", - "XBT", - "ARGO_FLOAT", - } + expedition = Expedition( + schedule=schedule, + instruments_config=get_expedition.instruments_config, + ship_config=get_expedition.ship_config, + ) + assert ( + set(instrument.name for instrument in expedition.get_instruments()) + == { + "CTD", + "UNDERWATER_ST", # not added above but underway instruments are auto present from instruments_config in expedition_dir/expedition.yaml + "ADCP", # as above + "ARGO_FLOAT", + "XBT", + } + ) @pytest.mark.parametrize( @@ -165,15 +180,15 @@ def test_verify_schedule_errors( @pytest.fixture -def schedule(tmp_file): +def expedition(tmp_file): with open(tmp_file, "w") as file: file.write(get_example_expedition()) - return Expedition.from_yaml(tmp_file).schedule + return Expedition.from_yaml(tmp_file) @pytest.fixture -def schedule_no_xbt(schedule): - for waypoint in schedule.waypoints: +def expedition_no_xbt(expedition): + for waypoint in expedition.schedule.waypoints: if waypoint.instrument and any( instrument.name == "XBT" for instrument in waypoint.instrument ): @@ -183,54 +198,57 @@ def schedule_no_xbt(schedule): if instrument.name != "XBT" ] - return schedule + return expedition @pytest.fixture -def instruments_config(tmp_file): - with open(tmp_file, "w") as file: - file.write(get_example_expedition()) - return Expedition.from_yaml(tmp_file).instruments_config +def instruments_config_no_xbt(expedition): + delattr(expedition.instruments_config, "xbt_config") + return expedition.instruments_config @pytest.fixture -def instruments_config_no_xbt(instruments_config): - delattr(instruments_config, "xbt_config") - return instruments_config +def instruments_config_no_ctd(expedition): + delattr(expedition.instruments_config, "ctd_config") + return expedition.instruments_config @pytest.fixture -def instruments_config_no_ctd(instruments_config): - delattr(instruments_config, "ctd_config") - return instruments_config +def instruments_config_no_ctd_bgc(expedition): + delattr(expedition.instruments_config, "ctd_bgc_config") + return expedition.instruments_config @pytest.fixture -def instruments_config_no_ctd_bgc(instruments_config): - delattr(instruments_config, "ctd_bgc_config") - return instruments_config +def instruments_config_no_argo_float(expedition): + delattr(expedition.instruments_config, "argo_float_config") + return expedition.instruments_config @pytest.fixture -def instruments_config_no_argo_float(instruments_config): - delattr(instruments_config, "argo_float_config") - return instruments_config +def instruments_config_no_drifter(expedition): + delattr(expedition.instruments_config, "drifter_config") + return expedition.instruments_config @pytest.fixture -def instruments_config_no_drifter(instruments_config): - delattr(instruments_config, "drifter_config") - return instruments_config +def instruments_config_no_adcp(expedition): + delattr(expedition.instruments_config, "adcp_config") + return expedition.instruments_config -def test_verify_instruments_config(instruments_config, schedule) -> None: - instruments_config.verify(schedule) +@pytest.fixture +def instruments_config_no_underwater_st(expedition): + delattr(expedition.instruments_config, "ship_underwater_st_config") + return expedition.instruments_config -def test_verify_instruments_config_no_instrument( - instruments_config, schedule_no_xbt -) -> None: - instruments_config.verify(schedule_no_xbt) +def test_verify_instruments_config(expedition) -> None: + expedition.instruments_config.verify(expedition) + + +def test_verify_instruments_config_no_instrument(expedition, expedition_no_xbt) -> None: + expedition.instruments_config.verify(expedition_no_xbt) @pytest.mark.parametrize( @@ -238,40 +256,52 @@ def test_verify_instruments_config_no_instrument( [ pytest.param( "instruments_config_no_xbt", - ConfigError, - "Schedule includes instrument 'XBT', but instruments_config does not provide configuration for it.", - id="ShipConfigNoXBT", + InstrumentsConfigError, + "Expedition includes instrument 'XBT', but instruments_config does not provide configuration for it.", + id="InstrumentsConfigNoXBT", ), pytest.param( "instruments_config_no_ctd", - ConfigError, - "Schedule includes instrument 'CTD', but instruments_config does not provide configuration for it.", - id="ShipConfigNoCTD", + InstrumentsConfigError, + "Expedition includes instrument 'CTD', but instruments_config does not provide configuration for it.", + id="InstrumentsConfigNoCTD", ), pytest.param( "instruments_config_no_ctd_bgc", - ConfigError, - "Schedule includes instrument 'CTD_BGC', but instruments_config does not provide configuration for it.", - id="ShipConfigNoCTD_BGC", + InstrumentsConfigError, + "Expedition includes instrument 'CTD_BGC', but instruments_config does not provide configuration for it.", + id="InstrumentsConfigNoCTD_BGC", ), pytest.param( "instruments_config_no_argo_float", - ConfigError, - "Schedule includes instrument 'ARGO_FLOAT', but instruments_config does not provide configuration for it.", - id="ShipConfigNoARGO_FLOAT", + InstrumentsConfigError, + "Expedition includes instrument 'ARGO_FLOAT', but instruments_config does not provide configuration for it.", + id="InstrumentsConfigNoARGO_FLOAT", ), pytest.param( "instruments_config_no_drifter", - ConfigError, - "Schedule includes instrument 'DRIFTER', but instruments_config does not provide configuration for it.", - id="ShipConfigNoDRIFTER", + InstrumentsConfigError, + "Expedition includes instrument 'DRIFTER', but instruments_config does not provide configuration for it.", + id="InstrumentsConfigNoDRIFTER", + ), + pytest.param( + "instruments_config_no_adcp", + InstrumentsConfigError, + r"Underway instrument config attribute\(s\) are missing from YAML\. Must be Config object or None\.", + id="InstrumentsConfigNoADCP", + ), + pytest.param( + "instruments_config_no_underwater_st", + InstrumentsConfigError, + r"Underway instrument config attribute\(s\) are missing from YAML\. Must be Config object or None\.", + id="InstrumentsConfigNoUNDERWATER_ST", ), ], ) def test_verify_instruments_config_errors( - request, schedule, instruments_config_fixture, error, match + request, expedition, instruments_config_fixture, error, match ) -> None: instruments_config = request.getfixturevalue(instruments_config_fixture) with pytest.raises(error, match=match): - instruments_config.verify(schedule) + instruments_config.verify(expedition) From 66aa4a52347fe7762b007d132f70daa9efd40545 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 27 Oct 2025 16:35:13 +0100 Subject: [PATCH 21/97] move instruments base classes out of models/ dir --- src/virtualship/instruments/adcp.py | 5 +- src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/master.py | 113 +++++++++++++++++++++++++ src/virtualship/models/instruments.py | 115 -------------------------- 4 files changed, 115 insertions(+), 120 deletions(-) delete mode 100644 src/virtualship/models/instruments.py diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index ccf48738..cf5af169 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -3,14 +3,11 @@ from typing import ClassVar import numpy as np -from parcels import FieldSet, ParticleSet, ScipyParticle, Variable +from parcels import FieldSet, ParticleSet, ScipyParticle, Variable from virtualship.models.instruments import InputDataset from virtualship.models.spacetime import Spacetime -## TODO: __init__.py will also need updating! -# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py - @dataclass class ADCP: diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 15a46234..73125838 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -4,8 +4,8 @@ from typing import ClassVar import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable +from parcels import FieldSet, JITParticle, ParticleSet, Variable from virtualship.models.instruments import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index 6e098988..43fe6a00 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -1,4 +1,14 @@ +import abc +from collections.abc import Callable +from datetime import timedelta from enum import Enum +from pathlib import Path + +import copernicusmarine +from yaspin import yaspin + +from virtualship.models.space_time_region import SpaceTimeRegion +from virtualship.utils import ship_spinner class InstrumentType(Enum): @@ -47,3 +57,106 @@ def get_instruments_registry(): for inst in InstrumentType if _input_class_map.get(inst.value) is not None } + + +# Base classes + + +class InputDataset(abc.ABC): + """Base class for instrument input datasets.""" + + def __init__( + self, + name: str, + latlon_buffer: float, + datetime_buffer: float, + min_depth: float, + max_depth: float, + data_dir: str, + credentials: dict, + space_time_region: SpaceTimeRegion, + ): + """Initialise input dataset.""" + self.name = name + self.latlon_buffer = latlon_buffer + self.datetime_buffer = datetime_buffer + self.min_depth = min_depth + self.max_depth = max_depth + self.data_dir = data_dir + self.credentials = credentials + self.space_time_region = space_time_region + + @abc.abstractmethod + def get_datasets_dict(self) -> dict: + """Get parameters for instrument's variable(s) specific data download.""" + ... + + def download_data(self) -> None: + """Download data for the instrument using copernicusmarine.""" + parameter_args = dict( + minimum_longitude=self.space_time_region.spatial_range.minimum_longitude + - self.latlon_buffer, + maximum_longitude=self.space_time_region.spatial_range.maximum_longitude + + self.latlon_buffer, + minimum_latitude=self.space_time_region.spatial_range.minimum_latitude + - self.latlon_buffer, + maximum_latitude=self.space_time_region.spatial_range.maximum_latitude + + self.latlon_buffer, + start_datetime=self.space_time_region.time_range.start_time, + end_datetime=self.space_time_region.time_range.end_time + + timedelta(days=self.datetime_buffer), + minimum_depth=abs(self.min_depth), + maximum_depth=abs(self.max_depth), + output_directory=self.data_dir, + username=self.credentials["username"], + password=self.credentials["password"], + overwrite=True, + coordinates_selection_method="outside", + ) + + datasets_args = self.get_datasets_dict() + + for dataset in datasets_args.values(): + download_args = {**parameter_args, **dataset} + copernicusmarine.subset(**download_args) + + +class Instrument(abc.ABC): + """Base class for instruments.""" + + def __init__( + self, + name: str, + config, + input_dataset: InputDataset, + kernels: list[Callable], + ): + """Initialise instrument.""" + self.name = name + self.config = config + self.input_data = input_dataset + self.kernels = kernels + + # def load_fieldset(self): + # """Load fieldset for simulation.""" + # # paths = self.input_data.get_fieldset_paths() + # ... + + def get_output_path(self, output_dir: Path) -> Path: + """Get output path for results.""" + return output_dir / f"{self.name}.zarr" + + def run(self): + """Run instrument simulation.""" + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate() + spinner.ok("✅") + + @abc.abstractmethod + def simulate(self): + """Simulate instrument measurements.""" + ... diff --git a/src/virtualship/models/instruments.py b/src/virtualship/models/instruments.py deleted file mode 100644 index 72d9237d..00000000 --- a/src/virtualship/models/instruments.py +++ /dev/null @@ -1,115 +0,0 @@ -import abc -from collections.abc import Callable -from datetime import timedelta -from pathlib import Path - -import copernicusmarine -from yaspin import yaspin - -from virtualship.models.space_time_region import SpaceTimeRegion -from virtualship.utils import ship_spinner - -# TODO: -# Discussion: Should each instrument manage its own data files for modularity, -# or should we consolidate downloads to minimize file duplication across instruments? -# Consider starting with per-instrument files for simplicity, and refactor later if needed. - - -class InputDataset(abc.ABC): - """Base class for instrument input datasets.""" - - def __init__( - self, - name: str, - latlon_buffer: float, - datetime_buffer: float, - min_depth: float, - max_depth: float, - data_dir: str, - credentials: dict, - space_time_region: SpaceTimeRegion, - ): - """Initialise input dataset.""" - self.name = name - self.latlon_buffer = latlon_buffer - self.datetime_buffer = datetime_buffer - self.min_depth = min_depth - self.max_depth = max_depth - self.data_dir = data_dir - self.credentials = credentials - self.space_time_region = space_time_region - - @abc.abstractmethod - def get_datasets_dict(self) -> dict: - """Get parameters for instrument's variable(s) specific data download.""" - ... - - def download_data(self) -> None: - """Download data for the instrument using copernicusmarine.""" - parameter_args = dict( - minimum_longitude=self.space_time_region.spatial_range.minimum_longitude - - self.latlon_buffer, - maximum_longitude=self.space_time_region.spatial_range.maximum_longitude - + self.latlon_buffer, - minimum_latitude=self.space_time_region.spatial_range.minimum_latitude - - self.latlon_buffer, - maximum_latitude=self.space_time_region.spatial_range.maximum_latitude - + self.latlon_buffer, - start_datetime=self.space_time_region.time_range.start_time, - end_datetime=self.space_time_region.time_range.end_time - + timedelta(days=self.datetime_buffer), - minimum_depth=abs(self.min_depth), - maximum_depth=abs(self.max_depth), - output_directory=self.data_dir, - username=self.credentials["username"], - password=self.credentials["password"], - overwrite=True, - coordinates_selection_method="outside", - ) - - datasets_args = self.get_datasets_dict() - - for dataset in datasets_args.values(): - download_args = {**parameter_args, **dataset} - copernicusmarine.subset(**download_args) - - -class Instrument(abc.ABC): - """Base class for instruments.""" - - def __init__( - self, - name: str, - config, - input_dataset: InputDataset, - kernels: list[Callable], - ): - """Initialise instrument.""" - self.name = name - self.config = config - self.input_data = input_dataset - self.kernels = kernels - - # def load_fieldset(self): - # """Load fieldset for simulation.""" - # # paths = self.input_data.get_fieldset_paths() - # ... - - def get_output_path(self, output_dir: Path) -> Path: - """Get output path for results.""" - return output_dir / f"{self.name}.zarr" - - def run(self): - """Run instrument simulation.""" - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: - self.simulate() - spinner.ok("✅") - - @abc.abstractmethod - def simulate(self): - """Simulate instrument measurements.""" - ... From 06ddf37e636775bfc01fa9201d749310b50c96e4 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 27 Oct 2025 16:35:53 +0100 Subject: [PATCH 22/97] update base class imports --- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 4 +- src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 4 +- .../instruments/ship_underwater_st.py | 4 +- src/virtualship/instruments/xbt.py | 4 +- src/virtualship/models/expedition.py | 12 ++- tests/models/test_instruments.py | 97 ------------------- 9 files changed, 20 insertions(+), 111 deletions(-) delete mode 100644 tests/models/test_instruments.py diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index cf5af169..b80bb544 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -5,7 +5,7 @@ import numpy as np from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.models.instruments import InputDataset +from virtualship.instruments.master import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index b9949fb1..e9cadb8d 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -5,6 +5,7 @@ from typing import ClassVar import numpy as np + from parcels import ( AdvectionRK4, FieldSet, @@ -13,8 +14,7 @@ StatusCode, Variable, ) - -from virtualship.models.instruments import InputDataset +from virtualship.instruments.master import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 73125838..6a27d175 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -6,7 +6,7 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models.instruments import InputDataset +from virtualship.instruments.master import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 96c0efa7..74b5b81c 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -6,7 +6,7 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models.instruments import InputDataset +from virtualship.instruments.master import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 60ba5558..40aad9d4 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -4,9 +4,9 @@ from typing import ClassVar import numpy as np -from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models.instruments import InputDataset +from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable +from virtualship.instruments.master import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 332c0b2c..3c7bc0ee 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -3,9 +3,9 @@ from typing import ClassVar import numpy as np -from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.models.instruments import InputDataset +from parcels import FieldSet, ParticleSet, ScipyParticle, Variable +from virtualship.instruments.master import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 81cba7b9..4351d08e 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -4,9 +4,9 @@ from typing import ClassVar import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models.instruments import InputDataset +from parcels import FieldSet, JITParticle, ParticleSet, Variable +from virtualship.instruments.master import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 7e206e8f..1b501119 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -9,7 +9,6 @@ import yaml from virtualship.errors import InstrumentsConfigError, ScheduleError -from virtualship.instruments.master import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta from .location import Location @@ -17,8 +16,8 @@ if TYPE_CHECKING: from parcels import FieldSet - from virtualship.expedition.input_data import InputData + from virtualship.instruments.master import InstrumentType projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") @@ -45,8 +44,10 @@ def from_yaml(cls, file_path: str) -> Expedition: data = yaml.safe_load(file) return Expedition(**data) - def get_instruments(self) -> set[InstrumentType]: + def get_instruments(self): """Return a set of unique InstrumentType enums used in the expedition.""" + from virtualship.instruments.master import InstrumentType + instruments_in_expedition = [] # from waypoints for waypoint in self.schedule.waypoints: @@ -114,6 +115,8 @@ def verify( """ print("\nVerifying route... ") + from virtualship.instruments.master import InstrumentType + if check_space_time_region and self.space_time_region is None: raise ScheduleError( "space_time_region not found in schedule, please define it to fetch the data." @@ -180,6 +183,7 @@ def verify( # check that ship will arrive on time at each waypoint (in case no unexpected event happen) time = self.waypoints[0].time + for wp_i, (wp, wp_next) in enumerate( zip(self.waypoints, self.waypoints[1:], strict=False) ): @@ -414,6 +418,8 @@ def verify(self, expedition: Expedition) -> None: Removes instrument configs not present in the schedule and checks that all scheduled instruments are configured. Raises ConfigError if any scheduled instrument is missing a config. """ + from virtualship.instruments.master import InstrumentType + instruments_in_expedition = expedition.get_instruments() instrument_config_map = { InstrumentType.ARGO_FLOAT: "argo_float_config", diff --git a/tests/models/test_instruments.py b/tests/models/test_instruments.py deleted file mode 100644 index 2ad73cdf..00000000 --- a/tests/models/test_instruments.py +++ /dev/null @@ -1,97 +0,0 @@ -import datetime -from unittest.mock import patch - -import pytest - -from virtualship.models.instruments import InputDataset -from virtualship.models.space_time_region import ( - SpaceTimeRegion, - SpatialRange, - TimeRange, -) - - -class DummyInputDataset(InputDataset): - """A minimal InputDataset subclass for testing purposes.""" - - def get_datasets_dict(self): - """Return a dummy datasets dict for testing.""" - return { - "dummy": { - "dataset_id": "test_id", - "variables": ["var1"], - "output_filename": "dummy.nc", - } - } - - -@pytest.fixture -def dummy_space_time_region(): - spatial_range = SpatialRange( - minimum_longitude=0, - maximum_longitude=1, - minimum_latitude=0, - maximum_latitude=1, - minimum_depth=0, - maximum_depth=10, - ) - base_time = datetime.datetime.strptime("1950-01-01", "%Y-%m-%d") - time_range = TimeRange( - start_time=base_time, - end_time=base_time + datetime.timedelta(hours=1), - ) - return SpaceTimeRegion( - spatial_range=spatial_range, - time_range=time_range, - ) - - -def test_inputdataset_abstract_instantiation(): - # instantiation should not be allowed - with pytest.raises(TypeError): - InputDataset( - name="test", - latlon_buffer=0, - datetime_buffer=0, - min_depth=0, - max_depth=10, - data_dir=".", - credentials={"username": "u", "password": "p"}, - space_time_region=None, - ) - - -def test_dummyinputdataset_initialization(dummy_space_time_region): - ds = DummyInputDataset( - name="test", - latlon_buffer=0.5, - datetime_buffer=1, - min_depth=0, - max_depth=10, - data_dir=".", - credentials={"username": "u", "password": "p"}, - space_time_region=dummy_space_time_region, - ) - assert ds.name == "test" - assert ds.latlon_buffer == 0.5 - assert ds.datetime_buffer == 1 - assert ds.min_depth == 0 - assert ds.max_depth == 10 - assert ds.data_dir == "." - assert ds.credentials["username"] == "u" - - -@patch("virtualship.models.instruments.copernicusmarine.subset") -def test_download_data_calls_subset(mock_subset, dummy_space_time_region): - ds = DummyInputDataset( - name="test", - latlon_buffer=0.5, - datetime_buffer=1, - min_depth=0, - max_depth=10, - data_dir=".", - credentials={"username": "u", "password": "p"}, - space_time_region=dummy_space_time_region, - ) - ds.download_data() - assert mock_subset.called From 5887177fd9df2e0387a2a5e32d7ede4457196a51 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 27 Oct 2025 16:36:20 +0100 Subject: [PATCH 23/97] make get_instruments_registry more robust with testing --- src/virtualship/instruments/master.py | 4 +- tests/instruments/test_master.py | 109 ++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 tests/instruments/test_master.py diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index 43fe6a00..2139e8d5 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -55,12 +55,14 @@ def get_instruments_registry(): "input_class": _input_class_map.get(inst.value), } for inst in InstrumentType - if _input_class_map.get(inst.value) is not None } # Base classes +# TODO: could InputDataset and Instrument be unified? +# TODO: and all associated child classes... + class InputDataset(abc.ABC): """Base class for instrument input datasets.""" diff --git a/tests/instruments/test_master.py b/tests/instruments/test_master.py new file mode 100644 index 00000000..f84238d0 --- /dev/null +++ b/tests/instruments/test_master.py @@ -0,0 +1,109 @@ +import datetime +from unittest.mock import patch + +import pytest + +from virtualship.instruments.master import ( + InputDataset, + InstrumentType, + get_instruments_registry, +) +from virtualship.models.space_time_region import ( + SpaceTimeRegion, + SpatialRange, + TimeRange, +) + + +class DummyInputDataset(InputDataset): + """A minimal InputDataset subclass for testing purposes.""" + + def get_datasets_dict(self): + """Return a dummy datasets dict for testing.""" + return { + "dummy": { + "dataset_id": "test_id", + "variables": ["var1"], + "output_filename": "dummy.nc", + } + } + + +@pytest.fixture +def dummy_space_time_region(): + spatial_range = SpatialRange( + minimum_longitude=0, + maximum_longitude=1, + minimum_latitude=0, + maximum_latitude=1, + minimum_depth=0, + maximum_depth=10, + ) + base_time = datetime.datetime.strptime("1950-01-01", "%Y-%m-%d") + time_range = TimeRange( + start_time=base_time, + end_time=base_time + datetime.timedelta(hours=1), + ) + return SpaceTimeRegion( + spatial_range=spatial_range, + time_range=time_range, + ) + + +def test_inputdataset_abstract_instantiation(): + # instantiation should not be allowed + with pytest.raises(TypeError): + InputDataset( + name="test", + latlon_buffer=0, + datetime_buffer=0, + min_depth=0, + max_depth=10, + data_dir=".", + credentials={"username": "u", "password": "p"}, + space_time_region=None, + ) + + +def test_dummyinputdataset_initialization(dummy_space_time_region): + ds = DummyInputDataset( + name="test", + latlon_buffer=0.5, + datetime_buffer=1, + min_depth=0, + max_depth=10, + data_dir=".", + credentials={"username": "u", "password": "p"}, + space_time_region=dummy_space_time_region, + ) + assert ds.name == "test" + assert ds.latlon_buffer == 0.5 + assert ds.datetime_buffer == 1 + assert ds.min_depth == 0 + assert ds.max_depth == 10 + assert ds.data_dir == "." + assert ds.credentials["username"] == "u" + + +@patch("virtualship.models.instruments.copernicusmarine.subset") +def test_download_data_calls_subset(mock_subset, dummy_space_time_region): + ds = DummyInputDataset( + name="test", + latlon_buffer=0.5, + datetime_buffer=1, + min_depth=0, + max_depth=10, + data_dir=".", + credentials={"username": "u", "password": "p"}, + space_time_region=dummy_space_time_region, + ) + ds.download_data() + assert mock_subset.called + + +def test_all_instruments_have_input_class(): + registry = get_instruments_registry() + for instrument in InstrumentType: + entry = registry.get(instrument) + assert entry is not None, f"No registry entry for {instrument}" + assert entry.get("input_class") is not None, f"No input_class for {instrument}" From cc35538de40aded688b3e346353f190a0f24417c Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 27 Oct 2025 17:23:40 +0100 Subject: [PATCH 24/97] update mock reanalysis period and refactor tests to use expedition fixture --- tests/cli/test_cli.py | 4 ++-- tests/cli/test_fetch.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 3d46787f..1b2c6e53 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -23,9 +23,9 @@ def fake_open_dataset(*args, **kwargs): "time": ( "time", [ - np.datetime64("1993-01-01"), np.datetime64("2022-01-01"), - ], # mock up rough renanalysis period + np.datetime64("2025-01-01"), + ], # mock up rough reanalysis period, covers test schedule ) } ) diff --git a/tests/cli/test_fetch.py b/tests/cli/test_fetch.py index 3ea56ea0..9725601a 100644 --- a/tests/cli/test_fetch.py +++ b/tests/cli/test_fetch.py @@ -38,8 +38,8 @@ def fake_open_dataset(*args, **kwargs): "time": ( "time", [ - np.datetime64("1993-01-01"), np.datetime64("2022-01-01"), + np.datetime64("2025-01-01"), ], # mock up rough renanalysis period ) } @@ -65,7 +65,7 @@ def expedition(tmpdir): @pytest.mark.usefixtures("copernicus_no_download") -def test_fetch(schedule, ship_config, tmpdir): +def test_fetch(expedition, tmpdir): """Test the fetch command, but mock the download and dataset metadata interrogation.""" _fetch(Path(tmpdir), "test", "test") @@ -99,12 +99,12 @@ def test_complete_download(tmp_path): @pytest.mark.usefixtures("copernicus_no_download") -def test_select_product_id(schedule): +def test_select_product_id(expedition): """Should return the physical reanalysis product id via the timings prescribed in the static schedule.yaml file.""" result = select_product_id( physical=True, - schedule_start=schedule.space_time_region.time_range.start_time, - schedule_end=schedule.space_time_region.time_range.end_time, + schedule_start=expedition.schedule.space_time_region.time_range.start_time, + schedule_end=expedition.schedule.space_time_region.time_range.end_time, username="test", password="test", ) @@ -112,12 +112,12 @@ def test_select_product_id(schedule): @pytest.mark.usefixtures("copernicus_no_download") -def test_start_end_in_product_timerange(schedule): +def test_start_end_in_product_timerange(expedition): """Should return True for valid range ass determined by the static schedule.yaml file.""" assert start_end_in_product_timerange( selected_id="cmems_mod_glo_phy_my_0.083deg_P1D-m", - schedule_start=schedule.space_time_region.time_range.start_time, - schedule_end=schedule.space_time_region.time_range.end_time, + schedule_start=expedition.schedule.space_time_region.time_range.start_time, + schedule_end=expedition.schedule.space_time_region.time_range.end_time, username="test", password="test", ) From 8f5af0466679b8a284f2b76c39c1ecfb4c863854 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 27 Oct 2025 17:41:55 +0100 Subject: [PATCH 25/97] refactor: reorganize instrument classes and update imports for clarity --- src/virtualship/cli/_fetch.py | 36 ++-- src/virtualship/cli/_plan.py | 2 +- src/virtualship/expedition/checkpoint.py | 2 +- .../expedition/simulate_schedule.py | 2 +- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/instruments/base.py | 109 ++++++++++++ src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 2 +- src/virtualship/instruments/master.py | 164 ------------------ .../instruments/ship_underwater_st.py | 2 +- src/virtualship/instruments/types.py | 18 ++ src/virtualship/instruments/xbt.py | 2 +- src/virtualship/models/expedition.py | 11 +- 15 files changed, 163 insertions(+), 195 deletions(-) create mode 100644 src/virtualship/instruments/base.py create mode 100644 src/virtualship/instruments/types.py diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 140163a6..17fb403a 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -12,7 +12,6 @@ from pydantic import BaseModel from virtualship.errors import CopernicusCatalogueError, IncompleteDownloadError -from virtualship.instruments.master import get_instruments_registry from virtualship.utils import ( _dump_yaml, _generic_load_yaml, @@ -29,8 +28,6 @@ DOWNLOAD_METADATA = "download_metadata.yaml" -INSTRUMENTS = get_instruments_registry() - def _fetch(path: str | Path, username: str | None, password: str | None) -> None: """ @@ -110,29 +107,44 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None coordinates_selection_method="outside", ) - # access instrument classes but keep only instruments which are in schedule - filter_instruments = { - k: v for k, v in INSTRUMENTS.items() if k in instruments_in_expedition + # Direct mapping from InstrumentType to input dataset class + from virtualship.instruments.adcp import ADCPInputDataset + from virtualship.instruments.argo_float import ArgoFloatInputDataset + from virtualship.instruments.ctd import CTDInputDataset + from virtualship.instruments.ctd_bgc import CTD_BGCInputDataset + from virtualship.instruments.drifter import DrifterInputDataset + from virtualship.instruments.ship_underwater_st import Underwater_STInputDataset + from virtualship.instruments.types import InstrumentType + from virtualship.instruments.xbt import XBTInputDataset + + INSTRUMENT_INPUT_DATASET_MAP = { + InstrumentType.CTD: CTDInputDataset, + InstrumentType.CTD_BGC: CTD_BGCInputDataset, + InstrumentType.DRIFTER: DrifterInputDataset, + InstrumentType.ARGO_FLOAT: ArgoFloatInputDataset, + InstrumentType.XBT: XBTInputDataset, + InstrumentType.ADCP: ADCPInputDataset, + InstrumentType.UNDERWATER_ST: Underwater_STInputDataset, } - # iterate across instruments and download data based on space_time_region - for itype, instrument in filter_instruments.items(): + # Only keep instruments present in the expedition + for itype in instruments_in_expedition: + input_dataset_class = INSTRUMENT_INPUT_DATASET_MAP.get(itype) + if input_dataset_class is None: + continue click.echo( f"\n\n{(' Fetching data for: ' + itype.value + ' ').center(80, '=')}\n\n" ) try: - input_dataset = instrument["input_class"]( + input_dataset = input_dataset_class( data_dir=download_folder, credentials=credentials, space_time_region=space_time_region, ) - input_dataset.download_data() - except InvalidUsernameOrPassword as e: shutil.rmtree(download_folder) raise e - click.echo(f"{itype.value} data download completed.") complete_download(download_folder) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index b6e44a70..a071c38e 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -29,7 +29,7 @@ type_to_textual, ) from virtualship.errors import UnexpectedError, UserError -from virtualship.instruments.master import InstrumentType +from virtualship.instruments.types import InstrumentType from virtualship.models import ( ADCPConfig, ArgoFloatConfig, diff --git a/src/virtualship/expedition/checkpoint.py b/src/virtualship/expedition/checkpoint.py index ff2dadd6..98fe1ae0 100644 --- a/src/virtualship/expedition/checkpoint.py +++ b/src/virtualship/expedition/checkpoint.py @@ -8,7 +8,7 @@ import yaml from virtualship.errors import CheckpointError -from virtualship.instruments.master import InstrumentType +from virtualship.instruments.types import InstrumentType from virtualship.models import Schedule diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 784e2d32..f8d142ea 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -11,7 +11,7 @@ from virtualship.instruments.ctd import CTD from virtualship.instruments.ctd_bgc import CTD_BGC from virtualship.instruments.drifter import Drifter -from virtualship.instruments.master import InstrumentType +from virtualship.instruments.types import InstrumentType from virtualship.instruments.xbt import XBT from virtualship.models import ( Expedition, diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index b80bb544..078effeb 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -5,7 +5,7 @@ import numpy as np from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.instruments.master import InputDataset +from virtualship.instruments.base import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index e9cadb8d..c654fd1e 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -14,7 +14,7 @@ StatusCode, Variable, ) -from virtualship.instruments.master import InputDataset +from virtualship.instruments.base import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py new file mode 100644 index 00000000..dfdb9c6a --- /dev/null +++ b/src/virtualship/instruments/base.py @@ -0,0 +1,109 @@ +import abc +from collections.abc import Callable +from datetime import timedelta +from pathlib import Path + +import copernicusmarine +import yaspin + +from virtualship.models import SpaceTimeRegion +from virtualship.utils import ship_spinner + + +class InputDataset(abc.ABC): + """Base class for instrument input datasets.""" + + def __init__( + self, + name: str, + latlon_buffer: float, + datetime_buffer: float, + min_depth: float, + max_depth: float, + data_dir: str, + credentials: dict, + space_time_region: SpaceTimeRegion, + ): + """Initialise input dataset.""" + self.name = name + self.latlon_buffer = latlon_buffer + self.datetime_buffer = datetime_buffer + self.min_depth = min_depth + self.max_depth = max_depth + self.data_dir = data_dir + self.credentials = credentials + self.space_time_region = space_time_region + + @abc.abstractmethod + def get_datasets_dict(self) -> dict: + """Get parameters for instrument's variable(s) specific data download.""" + ... + + def download_data(self) -> None: + """Download data for the instrument using copernicusmarine.""" + parameter_args = dict( + minimum_longitude=self.space_time_region.spatial_range.minimum_longitude + - self.latlon_buffer, + maximum_longitude=self.space_time_region.spatial_range.maximum_longitude + + self.latlon_buffer, + minimum_latitude=self.space_time_region.spatial_range.minimum_latitude + - self.latlon_buffer, + maximum_latitude=self.space_time_region.spatial_range.maximum_latitude + + self.latlon_buffer, + start_datetime=self.space_time_region.time_range.start_time, + end_datetime=self.space_time_region.time_range.end_time + + timedelta(days=self.datetime_buffer), + minimum_depth=abs(self.min_depth), + maximum_depth=abs(self.max_depth), + output_directory=self.data_dir, + username=self.credentials["username"], + password=self.credentials["password"], + overwrite=True, + coordinates_selection_method="outside", + ) + + datasets_args = self.get_datasets_dict() + for dataset in datasets_args.values(): + download_args = {**parameter_args, **dataset} + copernicusmarine.subset(**download_args) + + +class Instrument(abc.ABC): + """Base class for instruments.""" + + def __init__( + self, + name: str, + config, + input_dataset: InputDataset, + kernels: list[Callable], + ): + """Initialise instrument.""" + self.name = name + self.config = config + self.input_data = input_dataset + self.kernels = kernels + + # def load_fieldset(self): + # """Load fieldset for simulation.""" + # # paths = self.input_data.get_fieldset_paths() + # ... + + def get_output_path(self, output_dir: Path) -> Path: + """Get output path for results.""" + return output_dir / f"{self.name}.zarr" + + def run(self): + """Run instrument simulation.""" + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate() + spinner.ok("✅") + + @abc.abstractmethod + def simulate(self): + """Simulate instrument measurements.""" + ... diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 6a27d175..7d584601 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -6,7 +6,7 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.master import InputDataset +from virtualship.instruments.base import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 74b5b81c..1f45ce95 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -6,7 +6,7 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.master import InputDataset +from virtualship.instruments.base import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 40aad9d4..df765a0d 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -6,7 +6,7 @@ import numpy as np from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.master import InputDataset +from virtualship.instruments.base import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index 2139e8d5..e69de29b 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -1,164 +0,0 @@ -import abc -from collections.abc import Callable -from datetime import timedelta -from enum import Enum -from pathlib import Path - -import copernicusmarine -from yaspin import yaspin - -from virtualship.models.space_time_region import SpaceTimeRegion -from virtualship.utils import ship_spinner - - -class InstrumentType(Enum): - """Types of the instruments.""" - - # TODO: scope for this to evaporate in the future...? - - CTD = "CTD" - CTD_BGC = "CTD_BGC" - DRIFTER = "DRIFTER" - ARGO_FLOAT = "ARGO_FLOAT" - XBT = "XBT" - ADCP = "ADCP" - UNDERWATER_ST = "UNDERWATER_ST" - - @property - def is_underway(self) -> bool: - """Return True if instrument is an underway instrument (ADCP, UNDERWATER_ST).""" - return self in {InstrumentType.ADCP, InstrumentType.UNDERWATER_ST} - - -def get_instruments_registry(): - # local imports to avoid circular import issues - from virtualship.instruments.adcp import ADCPInputDataset - from virtualship.instruments.argo_float import ArgoFloatInputDataset - from virtualship.instruments.ctd import CTDInputDataset - from virtualship.instruments.ctd_bgc import CTD_BGCInputDataset - from virtualship.instruments.drifter import DrifterInputDataset - from virtualship.instruments.ship_underwater_st import Underwater_STInputDataset - from virtualship.instruments.xbt import XBTInputDataset - - _input_class_map = { - "CTD": CTDInputDataset, - "CTD_BGC": CTD_BGCInputDataset, - "DRIFTER": DrifterInputDataset, - "ARGO_FLOAT": ArgoFloatInputDataset, - "XBT": XBTInputDataset, - "ADCP": ADCPInputDataset, - "UNDERWATER_ST": Underwater_STInputDataset, - } - - return { - inst: { - "input_class": _input_class_map.get(inst.value), - } - for inst in InstrumentType - } - - -# Base classes - -# TODO: could InputDataset and Instrument be unified? -# TODO: and all associated child classes... - - -class InputDataset(abc.ABC): - """Base class for instrument input datasets.""" - - def __init__( - self, - name: str, - latlon_buffer: float, - datetime_buffer: float, - min_depth: float, - max_depth: float, - data_dir: str, - credentials: dict, - space_time_region: SpaceTimeRegion, - ): - """Initialise input dataset.""" - self.name = name - self.latlon_buffer = latlon_buffer - self.datetime_buffer = datetime_buffer - self.min_depth = min_depth - self.max_depth = max_depth - self.data_dir = data_dir - self.credentials = credentials - self.space_time_region = space_time_region - - @abc.abstractmethod - def get_datasets_dict(self) -> dict: - """Get parameters for instrument's variable(s) specific data download.""" - ... - - def download_data(self) -> None: - """Download data for the instrument using copernicusmarine.""" - parameter_args = dict( - minimum_longitude=self.space_time_region.spatial_range.minimum_longitude - - self.latlon_buffer, - maximum_longitude=self.space_time_region.spatial_range.maximum_longitude - + self.latlon_buffer, - minimum_latitude=self.space_time_region.spatial_range.minimum_latitude - - self.latlon_buffer, - maximum_latitude=self.space_time_region.spatial_range.maximum_latitude - + self.latlon_buffer, - start_datetime=self.space_time_region.time_range.start_time, - end_datetime=self.space_time_region.time_range.end_time - + timedelta(days=self.datetime_buffer), - minimum_depth=abs(self.min_depth), - maximum_depth=abs(self.max_depth), - output_directory=self.data_dir, - username=self.credentials["username"], - password=self.credentials["password"], - overwrite=True, - coordinates_selection_method="outside", - ) - - datasets_args = self.get_datasets_dict() - - for dataset in datasets_args.values(): - download_args = {**parameter_args, **dataset} - copernicusmarine.subset(**download_args) - - -class Instrument(abc.ABC): - """Base class for instruments.""" - - def __init__( - self, - name: str, - config, - input_dataset: InputDataset, - kernels: list[Callable], - ): - """Initialise instrument.""" - self.name = name - self.config = config - self.input_data = input_dataset - self.kernels = kernels - - # def load_fieldset(self): - # """Load fieldset for simulation.""" - # # paths = self.input_data.get_fieldset_paths() - # ... - - def get_output_path(self, output_dir: Path) -> Path: - """Get output path for results.""" - return output_dir / f"{self.name}.zarr" - - def run(self): - """Run instrument simulation.""" - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: - self.simulate() - spinner.ok("✅") - - @abc.abstractmethod - def simulate(self): - """Simulate instrument measurements.""" - ... diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 3c7bc0ee..fc4c1362 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -5,7 +5,7 @@ import numpy as np from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.instruments.master import InputDataset +from virtualship.instruments.base import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/types.py b/src/virtualship/instruments/types.py new file mode 100644 index 00000000..9ae221e9 --- /dev/null +++ b/src/virtualship/instruments/types.py @@ -0,0 +1,18 @@ +from enum import Enum + + +class InstrumentType(Enum): + """Types of the instruments.""" + + CTD = "CTD" + CTD_BGC = "CTD_BGC" + DRIFTER = "DRIFTER" + ARGO_FLOAT = "ARGO_FLOAT" + XBT = "XBT" + ADCP = "ADCP" + UNDERWATER_ST = "UNDERWATER_ST" + + @property + def is_underway(self) -> bool: + """Return True if instrument is an underway instrument (ADCP, UNDERWATER_ST).""" + return self in {InstrumentType.ADCP, InstrumentType.UNDERWATER_ST} diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 4351d08e..5d3b52ef 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -6,7 +6,7 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.master import InputDataset +from virtualship.instruments.base import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 1b501119..d7559fd0 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -9,6 +9,7 @@ import yaml from virtualship.errors import InstrumentsConfigError, ScheduleError +from virtualship.instruments.types import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta from .location import Location @@ -17,7 +18,6 @@ if TYPE_CHECKING: from parcels import FieldSet from virtualship.expedition.input_data import InputData - from virtualship.instruments.master import InstrumentType projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") @@ -44,10 +44,8 @@ def from_yaml(cls, file_path: str) -> Expedition: data = yaml.safe_load(file) return Expedition(**data) - def get_instruments(self): + def get_instruments(self) -> set[InstrumentType]: """Return a set of unique InstrumentType enums used in the expedition.""" - from virtualship.instruments.master import InstrumentType - instruments_in_expedition = [] # from waypoints for waypoint in self.schedule.waypoints: @@ -115,8 +113,6 @@ def verify( """ print("\nVerifying route... ") - from virtualship.instruments.master import InstrumentType - if check_space_time_region and self.space_time_region is None: raise ScheduleError( "space_time_region not found in schedule, please define it to fetch the data." @@ -183,7 +179,6 @@ def verify( # check that ship will arrive on time at each waypoint (in case no unexpected event happen) time = self.waypoints[0].time - for wp_i, (wp, wp_next) in enumerate( zip(self.waypoints, self.waypoints[1:], strict=False) ): @@ -418,8 +413,6 @@ def verify(self, expedition: Expedition) -> None: Removes instrument configs not present in the schedule and checks that all scheduled instruments are configured. Raises ConfigError if any scheduled instrument is missing a config. """ - from virtualship.instruments.master import InstrumentType - instruments_in_expedition = expedition.get_instruments() instrument_config_map = { InstrumentType.ARGO_FLOAT: "argo_float_config", From fdc0e6e4f23908cd954cf06f79dbbfd17261bcfe Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 27 Oct 2025 21:18:14 +0100 Subject: [PATCH 26/97] implement instrument registration and input dataset retrieval --- src/virtualship/cli/_fetch.py | 23 ++----------------- src/virtualship/instruments/adcp.py | 3 +++ src/virtualship/instruments/argo_float.py | 3 +++ src/virtualship/instruments/ctd.py | 3 +++ src/virtualship/instruments/ctd_bgc.py | 3 +++ src/virtualship/instruments/drifter.py | 3 +++ src/virtualship/instruments/master.py | 0 .../instruments/ship_underwater_st.py | 3 +++ src/virtualship/instruments/xbt.py | 3 +++ src/virtualship/utils.py | 15 ++++++++++++ 10 files changed, 38 insertions(+), 21 deletions(-) delete mode 100644 src/virtualship/instruments/master.py diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 17fb403a..50dda3a5 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -16,6 +16,7 @@ _dump_yaml, _generic_load_yaml, _get_expedition, + get_input_dataset_class, ) if TYPE_CHECKING: @@ -107,29 +108,9 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None coordinates_selection_method="outside", ) - # Direct mapping from InstrumentType to input dataset class - from virtualship.instruments.adcp import ADCPInputDataset - from virtualship.instruments.argo_float import ArgoFloatInputDataset - from virtualship.instruments.ctd import CTDInputDataset - from virtualship.instruments.ctd_bgc import CTD_BGCInputDataset - from virtualship.instruments.drifter import DrifterInputDataset - from virtualship.instruments.ship_underwater_st import Underwater_STInputDataset - from virtualship.instruments.types import InstrumentType - from virtualship.instruments.xbt import XBTInputDataset - - INSTRUMENT_INPUT_DATASET_MAP = { - InstrumentType.CTD: CTDInputDataset, - InstrumentType.CTD_BGC: CTD_BGCInputDataset, - InstrumentType.DRIFTER: DrifterInputDataset, - InstrumentType.ARGO_FLOAT: ArgoFloatInputDataset, - InstrumentType.XBT: XBTInputDataset, - InstrumentType.ADCP: ADCPInputDataset, - InstrumentType.UNDERWATER_ST: Underwater_STInputDataset, - } - # Only keep instruments present in the expedition for itype in instruments_in_expedition: - input_dataset_class = INSTRUMENT_INPUT_DATASET_MAP.get(itype) + input_dataset_class = get_input_dataset_class(itype) if input_dataset_class is None: continue click.echo( diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 078effeb..1bc67e00 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -6,7 +6,9 @@ from parcels import FieldSet, ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime +from virtualship.utils import register_instrument @dataclass @@ -32,6 +34,7 @@ def _sample_velocity(particle, fieldset, time): ) +@register_instrument(InstrumentType.ADCP) class ADCPInputDataset(InputDataset): """Input dataset for ADCP instrument.""" diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index c654fd1e..7f7d23a1 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -15,7 +15,9 @@ Variable, ) from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime +from virtualship.utils import register_instrument @dataclass @@ -116,6 +118,7 @@ def _check_error(particle, fieldset, time): particle.delete() +@register_instrument(InstrumentType.ARGO_FLOAT) class ArgoFloatInputDataset(InputDataset): """Input dataset for ArgoFloat instrument.""" diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 7d584601..cd8fd330 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -7,7 +7,9 @@ from parcels import FieldSet, JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime +from virtualship.utils import register_instrument @dataclass @@ -54,6 +56,7 @@ def _ctd_cast(particle, fieldset, time): particle.delete() +@register_instrument(InstrumentType.CTD) class CTDInputDataset(InputDataset): """Input dataset for CTD instrument.""" diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 1f45ce95..92f717db 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -7,7 +7,9 @@ from parcels import FieldSet, JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime +from virtualship.utils import register_instrument @dataclass @@ -79,6 +81,7 @@ def _ctd_bgc_cast(particle, fieldset, time): particle.delete() +@register_instrument(InstrumentType.CTD_BGC) class CTD_BGCInputDataset(InputDataset): """Input dataset object for CTD_BGC instrument.""" diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index df765a0d..4ca0d087 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -7,7 +7,9 @@ from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime +from virtualship.utils import register_instrument @dataclass @@ -41,6 +43,7 @@ def _check_lifetime(particle, fieldset, time): particle.delete() +@register_instrument(InstrumentType.DRIFTER) class DrifterInputDataset(InputDataset): """Input dataset for Drifter instrument.""" diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index fc4c1362..accfb5b3 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -6,7 +6,9 @@ from parcels import FieldSet, ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime +from virtualship.utils import register_instrument @dataclass @@ -34,6 +36,7 @@ def _sample_temperature(particle, fieldset, time): particle.T = fieldset.T[time, particle.depth, particle.lat, particle.lon] +@register_instrument(InstrumentType.UNDERWATER_ST) class Underwater_STInputDataset(InputDataset): """Input dataset for Underwater_ST instrument.""" diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 5d3b52ef..f5ec5fd0 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -7,7 +7,9 @@ from parcels import FieldSet, JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime +from virtualship.utils import register_instrument @dataclass @@ -55,6 +57,7 @@ def _xbt_cast(particle, fieldset, time): particle_ddepth = particle.max_depth - particle.depth +@register_instrument(InstrumentType.XBT) class XBTInputDataset(InputDataset): """Input dataset for XBT instrument.""" diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 0a39d035..e1054da1 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -239,3 +239,18 @@ def _get_expedition(expedition_dir: Path) -> Expedition: "🚢 ", ], ) + +# InstrumentType -> InputDataset registry and registration utilities. +INSTRUMENT_INPUT_DATASET_MAP = {} + + +def register_instrument(instrument_type): + def decorator(cls): + INSTRUMENT_INPUT_DATASET_MAP[instrument_type] = cls + return cls + + return decorator + + +def get_input_dataset_class(instrument_type): + return INSTRUMENT_INPUT_DATASET_MAP.get(instrument_type) From 95e6cbd0d47797f69dee2745094812e288666dd5 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 27 Oct 2025 21:44:24 +0100 Subject: [PATCH 27/97] refactor: reorganize imports in instrument test files for consistency --- src/virtualship/instruments/__init__.py | 20 +++++++++++++++++++ .../{test_master.py => test_base.py} | 16 ++++++--------- tests/instruments/test_ctd.py | 2 +- 3 files changed, 27 insertions(+), 11 deletions(-) rename tests/instruments/{test_master.py => test_base.py} (86%) diff --git a/src/virtualship/instruments/__init__.py b/src/virtualship/instruments/__init__.py index 5324da2c..b593ed38 100644 --- a/src/virtualship/instruments/__init__.py +++ b/src/virtualship/instruments/__init__.py @@ -1 +1,21 @@ """Instruments in VirtualShip.""" + +from . import ( + adcp, + argo_float, + ctd, + ctd_bgc, + drifter, + ship_underwater_st, + xbt, +) + +__all__ = [ + "adcp", + "argo_float", + "ctd", + "ctd_bgc", + "drifter", + "ship_underwater_st", + "xbt", +] diff --git a/tests/instruments/test_master.py b/tests/instruments/test_base.py similarity index 86% rename from tests/instruments/test_master.py rename to tests/instruments/test_base.py index f84238d0..f4192092 100644 --- a/tests/instruments/test_master.py +++ b/tests/instruments/test_base.py @@ -3,16 +3,14 @@ import pytest -from virtualship.instruments.master import ( - InputDataset, - InstrumentType, - get_instruments_registry, -) +from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.space_time_region import ( SpaceTimeRegion, SpatialRange, TimeRange, ) +from virtualship.utils import get_input_dataset_class class DummyInputDataset(InputDataset): @@ -85,7 +83,7 @@ def test_dummyinputdataset_initialization(dummy_space_time_region): assert ds.credentials["username"] == "u" -@patch("virtualship.models.instruments.copernicusmarine.subset") +@patch("virtualship.instruments.base.copernicusmarine.subset") def test_download_data_calls_subset(mock_subset, dummy_space_time_region): ds = DummyInputDataset( name="test", @@ -102,8 +100,6 @@ def test_download_data_calls_subset(mock_subset, dummy_space_time_region): def test_all_instruments_have_input_class(): - registry = get_instruments_registry() for instrument in InstrumentType: - entry = registry.get(instrument) - assert entry is not None, f"No registry entry for {instrument}" - assert entry.get("input_class") is not None, f"No input_class for {instrument}" + input_class = get_input_dataset_class(instrument) + assert input_class is not None, f"No input_class for {instrument}" diff --git a/tests/instruments/test_ctd.py b/tests/instruments/test_ctd.py index 14e0a276..0a8edcfa 100644 --- a/tests/instruments/test_ctd.py +++ b/tests/instruments/test_ctd.py @@ -9,8 +9,8 @@ import numpy as np import xarray as xr -from parcels import Field, FieldSet +from parcels import Field, FieldSet from virtualship.instruments.ctd import CTD, simulate_ctd from virtualship.models import Location, Spacetime From 01cee18ff6e52781ad9335cbbe4de512b265cca7 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 28 Oct 2025 11:50:21 +0100 Subject: [PATCH 28/97] further refactoring: instrument classes to use a unified InputDataset and Instrument structure --- src/virtualship/cli/_fetch.py | 2 +- src/virtualship/expedition/do_expedition.py | 81 +++--- src/virtualship/expedition/input_data.py | 255 ------------------ src/virtualship/instruments/adcp.py | 138 +++++----- src/virtualship/instruments/argo_float.py | 177 ++++++------ src/virtualship/instruments/base.py | 94 +++++-- src/virtualship/instruments/ctd.py | 193 +++++++------ src/virtualship/instruments/ctd_bgc.py | 231 ++++++++-------- src/virtualship/instruments/drifter.py | 180 ++++++------- .../instruments/ship_underwater_st.py | 124 ++++----- src/virtualship/instruments/xbt.py | 197 +++++++------- src/virtualship/models/expedition.py | 25 +- src/virtualship/utils.py | 22 +- 13 files changed, 746 insertions(+), 973 deletions(-) delete mode 100644 src/virtualship/expedition/input_data.py diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 50dda3a5..67695695 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -112,7 +112,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None for itype in instruments_in_expedition: input_dataset_class = get_input_dataset_class(itype) if input_dataset_class is None: - continue + raise RuntimeError(f"No input dataset class found for type {itype}.") click.echo( f"\n\n{(' Fetching data for: ' + itype.value + ' ').center(80, '=')}\n\n" ) diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index 921ea528..ef5b6037 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -6,17 +6,11 @@ import pyproj -from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash -from virtualship.models import Expedition, Schedule -from virtualship.utils import ( - CHECKPOINT, - _get_expedition, -) +from virtualship.models import Schedule +from virtualship.utils import CHECKPOINT, _get_expedition, get_instrument_class from .checkpoint import Checkpoint from .expedition_cost import expedition_cost -from .input_data import InputData -from .simulate_measurements import simulate_measurements from .simulate_schedule import ScheduleProblem, simulate_schedule # projection used to sail between waypoints @@ -51,6 +45,7 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> checkpoint.verify(expedition.schedule) # load fieldsets + _load_input_data = [] # TEMPORARY! loaded_input_data = _load_input_data( expedition_dir=expedition_dir, expedition=expedition, @@ -106,56 +101,42 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> # simulate measurements print("\nSimulating measurements. This may take a while...\n") - simulate_measurements( - expedition_dir, - expedition.instruments_config, - loaded_input_data, - schedule_results.measurements_to_simulate, - ) - print("\nAll measurement simulations are complete.") - print("\n----- EXPEDITION RESULTS ------") - print("\nYour expedition has concluded successfully!") - print( - f"Your measurements can be found in the '{expedition_dir}/results' directory." - ) - print("\n------------- END -------------\n") + # TODO: this is where XYZInstrument.run() could be called instead of simulate_measurements!? + # TODO: this time maybe looping through measurements to simulate in some form... + # TODO: first in explicit per instrument, then think about whether can be automated more...not the end of the world if just have to explain in documentation that changes must be made here... + instruments_in_expedition = expedition.get_instruments() -def _load_input_data( - expedition_dir: Path, - expedition: Expedition, - input_data: Path | None, -) -> InputData: - """ - Load the input data. + for itype in instruments_in_expedition: + instrument_class = get_instrument_class(itype) + if instrument_class is None: + raise RuntimeError(f"No instrument class found for type {itype}.") - :param expedition_dir: Directory of the expedition. - :param expedition: Expedition object. - :param input_data: Folder containing input data. - :return: InputData object. - """ - if input_data is None: - space_time_region_hash = get_space_time_region_hash( - expedition.schedule.space_time_region + measurements = schedule_results.measurements_to_simulate.get(itype.name.lower()) + + instrument_class.run( + expedition_dir.joinpath("results", f"{itype.name.lower()}.zarr"), + measurements=measurements, + fieldset=loaded_input_data.get_fieldset_for_instrument_type(itype), + expedition=expedition, ) - input_data = get_existing_download(expedition_dir, space_time_region_hash) - assert input_data is not None, ( - "Input data hasn't been found. Have you run the `virtualship fetch` command?" - ) + # simulate_measurements( + # expedition_dir, + # expedition.instruments_config, + # loaded_input_data, + # schedule_results.measurements_to_simulate, + # ) - return InputData.load( - directory=input_data, - load_adcp=expedition.instruments_config.adcp_config is not None, - load_argo_float=expedition.instruments_config.argo_float_config is not None, - load_ctd=expedition.instruments_config.ctd_config is not None, - load_ctd_bgc=expedition.instruments_config.ctd_bgc_config is not None, - load_drifter=expedition.instruments_config.drifter_config is not None, - load_xbt=expedition.instruments_config.xbt_config is not None, - load_ship_underwater_st=expedition.instruments_config.ship_underwater_st_config - is not None, + print("\nAll measurement simulations are complete.") + + print("\n----- EXPEDITION RESULTS ------") + print("\nYour expedition has concluded successfully!") + print( + f"Your measurements can be found in the '{expedition_dir}/results' directory." ) + print("\n------------- END -------------\n") def _load_checkpoint(expedition_dir: Path) -> Checkpoint | None: diff --git a/src/virtualship/expedition/input_data.py b/src/virtualship/expedition/input_data.py deleted file mode 100644 index fa48e0a7..00000000 --- a/src/virtualship/expedition/input_data.py +++ /dev/null @@ -1,255 +0,0 @@ -"""InputData class.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - -from parcels import Field, FieldSet - - -@dataclass -class InputData: - """A collection of fieldsets that function as input data for simulation.""" - - adcp_fieldset: FieldSet | None - argo_float_fieldset: FieldSet | None - ctd_fieldset: FieldSet | None - ctd_bgc_fieldset: FieldSet | None - drifter_fieldset: FieldSet | None - xbt_fieldset: FieldSet | None - ship_underwater_st_fieldset: FieldSet | None - - @classmethod - def load( - cls, - directory: str | Path, - load_adcp: bool, - load_argo_float: bool, - load_ctd: bool, - load_ctd_bgc: bool, - load_drifter: bool, - load_xbt: bool, - load_ship_underwater_st: bool, - ) -> InputData: - """ - Create an instance of this class from netCDF files. - - For now this function makes a lot of assumption about file location and contents. - - :param directory: Input data directory. - :param load_adcp: Whether to load the ADCP fieldset. - :param load_argo_float: Whether to load the argo float fieldset. - :param load_ctd: Whether to load the CTD fieldset. - :param load_ctd_bgc: Whether to load the CTD BGC fieldset. - :param load_drifter: Whether to load the drifter fieldset. - :param load_ship_underwater_st: Whether to load the ship underwater ST fieldset. - :returns: An instance of this class with loaded fieldsets. - """ - directory = Path(directory) - if load_drifter: - drifter_fieldset = cls._load_drifter_fieldset(directory) - else: - drifter_fieldset = None - if load_argo_float: - argo_float_fieldset = cls._load_argo_float_fieldset(directory) - else: - argo_float_fieldset = None - if load_ctd_bgc: - ctd_bgc_fieldset = cls._load_ctd_bgc_fieldset(directory) - else: - ctd_bgc_fieldset = None - if load_adcp or load_ctd or load_ship_underwater_st or load_xbt: - ship_fieldset = cls._load_ship_fieldset(directory) - if load_adcp: - adcp_fieldset = ship_fieldset - else: - adcp_fieldset = None - if load_ctd: - ctd_fieldset = ship_fieldset - else: - ctd_fieldset = None - if load_ship_underwater_st: - ship_underwater_st_fieldset = ship_fieldset - else: - ship_underwater_st_fieldset = None - if load_xbt: - xbt_fieldset = ship_fieldset - else: - xbt_fieldset = None - - return InputData( - adcp_fieldset=adcp_fieldset, - argo_float_fieldset=argo_float_fieldset, - ctd_fieldset=ctd_fieldset, - ctd_bgc_fieldset=ctd_bgc_fieldset, - drifter_fieldset=drifter_fieldset, - xbt_fieldset=xbt_fieldset, - ship_underwater_st_fieldset=ship_underwater_st_fieldset, - ) - - @classmethod - def _load_ship_fieldset(cls, directory: Path) -> FieldSet: - filenames = { - "U": directory.joinpath("ship_uv.nc"), - "V": directory.joinpath("ship_uv.nc"), - "S": directory.joinpath("ship_s.nc"), - "T": directory.joinpath("ship_t.nc"), - } - variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} - dimensions = { - "lon": "longitude", - "lat": "latitude", - "time": "time", - "depth": "depth", - } - - # create the fieldset and set interpolation methods - fieldset = FieldSet.from_netcdf( - filenames, variables, dimensions, allow_time_extrapolation=True - ) - fieldset.T.interp_method = "linear_invdist_land_tracer" - fieldset.S.interp_method = "linear_invdist_land_tracer" - - # make depth negative - for g in fieldset.gridset.grids: - g.negate_depth() - - # add bathymetry data - bathymetry_file = directory.joinpath("bathymetry.nc") - bathymetry_variables = ("bathymetry", "deptho") - bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} - bathymetry_field = Field.from_netcdf( - bathymetry_file, bathymetry_variables, bathymetry_dimensions - ) - # make depth negative - bathymetry_field.data = -bathymetry_field.data - fieldset.add_field(bathymetry_field) - - # read in data already - fieldset.computeTimeChunk(0, 1) - - return fieldset - - @classmethod - def _load_ctd_bgc_fieldset(cls, directory: Path) -> FieldSet: - filenames = { - "U": directory.joinpath("ship_uv.nc"), - "V": directory.joinpath("ship_uv.nc"), - "o2": directory.joinpath("ctd_bgc_o2.nc"), - "chl": directory.joinpath("ctd_bgc_chl.nc"), - "no3": directory.joinpath("ctd_bgc_no3.nc"), - "po4": directory.joinpath("ctd_bgc_po4.nc"), - "ph": directory.joinpath("ctd_bgc_ph.nc"), - "phyc": directory.joinpath("ctd_bgc_phyc.nc"), - "nppv": directory.joinpath("ctd_bgc_nppv.nc"), - } - variables = { - "U": "uo", - "V": "vo", - "o2": "o2", - "chl": "chl", - "no3": "no3", - "po4": "po4", - "ph": "ph", - "phyc": "phyc", - "nppv": "nppv", - } - dimensions = { - "lon": "longitude", - "lat": "latitude", - "time": "time", - "depth": "depth", - } - - fieldset = FieldSet.from_netcdf( - filenames, variables, dimensions, allow_time_extrapolation=True - ) - fieldset.o2.interp_method = "linear_invdist_land_tracer" - fieldset.chl.interp_method = "linear_invdist_land_tracer" - fieldset.no3.interp_method = "linear_invdist_land_tracer" - fieldset.po4.interp_method = "linear_invdist_land_tracer" - fieldset.ph.interp_method = "linear_invdist_land_tracer" - fieldset.phyc.interp_method = "linear_invdist_land_tracer" - fieldset.nppv.interp_method = "linear_invdist_land_tracer" - - # make depth negative - for g in fieldset.gridset.grids: - g.negate_depth() - - # add bathymetry data - bathymetry_file = directory.joinpath("bathymetry.nc") - bathymetry_variables = ("bathymetry", "deptho") - bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} - bathymetry_field = Field.from_netcdf( - bathymetry_file, bathymetry_variables, bathymetry_dimensions - ) - # make depth negative - bathymetry_field.data = -bathymetry_field.data - fieldset.add_field(bathymetry_field) - - # read in data already - fieldset.computeTimeChunk(0, 1) - - return fieldset - - @classmethod - def _load_drifter_fieldset(cls, directory: Path) -> FieldSet: - filenames = { - "U": directory.joinpath("drifter_uv.nc"), - "V": directory.joinpath("drifter_uv.nc"), - "T": directory.joinpath("drifter_t.nc"), - } - variables = {"U": "uo", "V": "vo", "T": "thetao"} - dimensions = { - "lon": "longitude", - "lat": "latitude", - "time": "time", - "depth": "depth", - } - - fieldset = FieldSet.from_netcdf( - filenames, variables, dimensions, allow_time_extrapolation=False - ) - fieldset.T.interp_method = "linear_invdist_land_tracer" - - # make depth negative - for g in fieldset.gridset.grids: - g.negate_depth() - - # read in data already - fieldset.computeTimeChunk(0, 1) - - return fieldset - - @classmethod - def _load_argo_float_fieldset(cls, directory: Path) -> FieldSet: - filenames = { - "U": directory.joinpath("argo_float_uv.nc"), - "V": directory.joinpath("argo_float_uv.nc"), - "S": directory.joinpath("argo_float_s.nc"), - "T": directory.joinpath("argo_float_t.nc"), - } - variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} - dimensions = { - "lon": "longitude", - "lat": "latitude", - "time": "time", - "depth": "depth", - } - - fieldset = FieldSet.from_netcdf( - filenames, variables, dimensions, allow_time_extrapolation=False - ) - fieldset.T.interp_method = "linear_invdist_land_tracer" - fieldset.S.interp_method = "linear_invdist_land_tracer" - - # make depth negative - for g in fieldset.gridset.grids: - if max(g.depth) > 0: - g.negate_depth() - - # read in data already - fieldset.computeTimeChunk(0, 1) - - return fieldset diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 1bc67e00..5caab5bf 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -4,11 +4,11 @@ import numpy as np -from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.instruments.base import InputDataset +from parcels import ParticleSet, ScipyParticle, Variable +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.utils import register_input_dataset, register_instrument @dataclass @@ -34,7 +34,7 @@ def _sample_velocity(particle, fieldset, time): ) -@register_instrument(InstrumentType.ADCP) +@register_input_dataset(InstrumentType.ADCP) class ADCPInputDataset(InputDataset): """Input dataset for ADCP instrument.""" @@ -69,78 +69,64 @@ def get_datasets_dict(self) -> dict: } -# TODO: uncomment when ready for new simulation logic! -# class ADCPInstrument(instruments.Instrument): -# """ADCP instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(ADCP.name, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -# TODO: to be replaced with new simulation logic -## -- old simulation code - - -def simulate_adcp( - fieldset: FieldSet, - out_path: str | Path, - max_depth: float, - min_depth: float, - num_bins: int, - sample_points: list[Spacetime], -) -> None: - """ - Use Parcels to simulate an ADCP in a fieldset. - - :param fieldset: The fieldset to simulate the ADCP in. - :param out_path: The path to write the results to. - :param max_depth: Maximum depth the ADCP can measure. - :param min_depth: Minimum depth the ADCP can measure. - :param num_bins: How many samples to take in the complete range between max_depth and min_depth. - :param sample_points: The places and times to sample at. - """ - sample_points.sort(key=lambda p: p.time) - - bins = np.linspace(max_depth, min_depth, num_bins) - num_particles = len(bins) - particleset = ParticleSet.from_list( - fieldset=fieldset, - pclass=_ADCPParticle, - lon=np.full( - num_particles, 0.0 - ), # initial lat/lon are irrelevant and will be overruled later. - lat=np.full(num_particles, 0.0), - depth=bins, - time=0, # same for time - ) - - # define output file for the simulation - # outputdt set to infinite as we just want to write at the end of every call to 'execute' - out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) - - for point in sample_points: - particleset.lon_nextloop[:] = point.location.lon - particleset.lat_nextloop[:] = point.location.lat - particleset.time_nextloop[:] = fieldset.time_origin.reltime( - np.datetime64(point.time) +@register_instrument(InstrumentType.ADCP) +class ADCPInstrument(Instrument): + """ADCP instrument class.""" + + def __init__( + self, + input_dataset: InputDataset, + ): + """Initialize ADCPInstrument.""" + filenames = { + "UV": input_dataset.data_dir.joinpath(f"{input_dataset.name}_uv.nc"), + } + variables = {"UV": ["uo", "vo"]} + super().__init__( + input_dataset, + filenames, + variables, + add_bathymetry=False, + allow_time_extrapolation=True, ) - # perform one step using the particleset - # dt and runtime are set so exactly one step is made. - particleset.execute( - [_sample_velocity], - dt=1, - runtime=1, - verbose_progress=False, - output_file=out_file, + def simulate( + self, + out_path: str | Path, + max_depth: float, + min_depth: float, + num_bins: int, + sample_points: list[Spacetime], + ) -> None: + """Simulate ADCP measurements.""" + sample_points.sort(key=lambda p: p.time) + + fieldset = self.load_input_data() + + bins = np.linspace(max_depth, min_depth, num_bins) + num_particles = len(bins) + particleset = ParticleSet.from_list( + fieldset=fieldset, + pclass=_ADCPParticle, + lon=np.full(num_particles, 0.0), + lat=np.full(num_particles, 0.0), + depth=bins, + time=0, ) + + out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) + + for point in sample_points: + particleset.lon_nextloop[:] = point.location.lon + particleset.lat_nextloop[:] = point.location.lat + particleset.time_nextloop[:] = fieldset.time_origin.reltime( + np.datetime64(point.time) + ) + + particleset.execute( + [_sample_velocity], + dt=1, + runtime=1, + verbose_progress=False, + output_file=out_file, + ) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 7f7d23a1..66b25bdf 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -8,16 +8,15 @@ from parcels import ( AdvectionRK4, - FieldSet, JITParticle, ParticleSet, StatusCode, Variable, ) -from virtualship.instruments.base import InputDataset +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.utils import register_input_dataset, register_instrument @dataclass @@ -118,7 +117,7 @@ def _check_error(particle, fieldset, time): particle.delete() -@register_instrument(InstrumentType.ARGO_FLOAT) +@register_input_dataset(InstrumentType.ARGO_FLOAT) class ArgoFloatInputDataset(InputDataset): """Input dataset for ArgoFloat instrument.""" @@ -163,89 +162,89 @@ def get_datasets_dict(self) -> dict: } -# class ArgoFloatInstrument(instruments.Instrument): -# """ArgoFloat instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(ArgoFloat.name, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -def simulate_argo_floats( - fieldset: FieldSet, - out_path: str | Path, - argo_floats: list[ArgoFloat], - outputdt: timedelta, - endtime: datetime | None, -) -> None: - """ - Use Parcels to simulate a set of Argo floats in a fieldset. - - :param fieldset: The fieldset to simulate the Argo floats in. - :param out_path: The path to write the results to. - :param argo_floats: A list of Argo floats to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :param endtime: Stop at this time, or if None, continue until the end of the fieldset. - """ - DT = 10.0 # dt of Argo float simulation integrator - - if len(argo_floats) == 0: - print( - "No Argo floats provided. Parcels currently crashes when providing an empty particle set, so no argo floats simulation will be done and no files will be created." +@register_instrument(InstrumentType.ARGO_FLOAT) +class ArgoFloatInstrument(Instrument): + """ArgoFloat instrument class.""" + + def __init__( + self, + input_dataset: InputDataset, + ): + """Initialize ArgoFloatInstrument.""" + filenames = { + "UV": input_dataset.data_dir.joinpath("argo_float_uv.nc"), + "S": input_dataset.data_dir.joinpath("argo_float_s.nc"), + "T": input_dataset.data_dir.joinpath("argo_float_t.nc"), + } + variables = {"UV": ["uo", "vo"], "S": "so", "T": "thetao"} + super().__init__( + input_dataset, + filenames, + variables, + add_bathymetry=False, + allow_time_extrapolation=False, + ) + + def simulate( + self, + argo_floats: list[ArgoFloat], + out_path: str | Path, + outputdt: timedelta, + endtime: datetime | None = None, + ) -> None: + """Simulate Argo float measurements.""" + DT = 10.0 # dt of Argo float simulation integrator + + if len(argo_floats) == 0: + print( + "No Argo floats provided. Parcels currently crashes when providing an empty particle set, so no argo floats simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + fieldset = self.load_input_data() + + # define parcel particles + argo_float_particleset = ParticleSet( + fieldset=fieldset, + pclass=_ArgoParticle, + lat=[argo.spacetime.location.lat for argo in argo_floats], + lon=[argo.spacetime.location.lon for argo in argo_floats], + depth=[argo.min_depth for argo in argo_floats], + time=[argo.spacetime.time for argo in argo_floats], + min_depth=[argo.min_depth for argo in argo_floats], + max_depth=[argo.max_depth for argo in argo_floats], + drift_depth=[argo.drift_depth for argo in argo_floats], + vertical_speed=[argo.vertical_speed for argo in argo_floats], + cycle_days=[argo.cycle_days for argo in argo_floats], + drift_days=[argo.drift_days for argo in argo_floats], + ) + + # define output file for the simulation + out_file = argo_float_particleset.ParticleFile( + name=out_path, outputdt=outputdt, chunks=[len(argo_float_particleset), 100] + ) + + # get earliest between fieldset end time and provide end time + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + if endtime is None: + actual_endtime = fieldset_endtime + elif endtime > fieldset_endtime: + print("WARN: Requested end time later than fieldset end time.") + actual_endtime = fieldset_endtime + else: + actual_endtime = np.timedelta64(endtime) + + # execute simulation + argo_float_particleset.execute( + [ + _argo_float_vertical_movement, + AdvectionRK4, + _keep_at_surface, + _check_error, + ], + endtime=actual_endtime, + dt=DT, + output_file=out_file, + verbose_progress=True, ) - # TODO when Parcels supports it this check can be removed. - return - - # define parcel particles - argo_float_particleset = ParticleSet( - fieldset=fieldset, - pclass=_ArgoParticle, - lat=[argo.spacetime.location.lat for argo in argo_floats], - lon=[argo.spacetime.location.lon for argo in argo_floats], - depth=[argo.min_depth for argo in argo_floats], - time=[argo.spacetime.time for argo in argo_floats], - min_depth=[argo.min_depth for argo in argo_floats], - max_depth=[argo.max_depth for argo in argo_floats], - drift_depth=[argo.drift_depth for argo in argo_floats], - vertical_speed=[argo.vertical_speed for argo in argo_floats], - cycle_days=[argo.cycle_days for argo in argo_floats], - drift_days=[argo.drift_days for argo in argo_floats], - ) - - # define output file for the simulation - out_file = argo_float_particleset.ParticleFile( - name=out_path, outputdt=outputdt, chunks=[len(argo_float_particleset), 100] - ) - - # get earliest between fieldset end time and provide end time - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - if endtime is None: - actual_endtime = fieldset_endtime - elif endtime > fieldset_endtime: - print("WARN: Requested end time later than fieldset end time.") - actual_endtime = fieldset_endtime - else: - actual_endtime = np.timedelta64(endtime) - - # execute simulation - argo_float_particleset.execute( - [ - _argo_float_vertical_movement, - AdvectionRK4, - _keep_at_surface, - _check_error, - ], - endtime=actual_endtime, - dt=DT, - output_file=out_file, - verbose_progress=True, - ) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index dfdb9c6a..2fab4795 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,11 +1,10 @@ import abc -from collections.abc import Callable from datetime import timedelta -from pathlib import Path import copernicusmarine import yaspin +from parcels import Field, FieldSet from virtualship.models import SpaceTimeRegion from virtualship.utils import ship_spinner @@ -37,7 +36,6 @@ def __init__( @abc.abstractmethod def get_datasets_dict(self) -> dict: """Get parameters for instrument's variable(s) specific data download.""" - ... def download_data(self) -> None: """Download data for the instrument using copernicusmarine.""" @@ -69,41 +67,89 @@ def download_data(self) -> None: class Instrument(abc.ABC): - """Base class for instruments.""" + """Base class for instruments and their simulation.""" def __init__( self, - name: str, - config, input_dataset: InputDataset, - kernels: list[Callable], + filenames: dict, + variables: dict, + add_bathymetry: bool, + allow_time_extrapolation: bool, + bathymetry_file: str = "bathymetry.nc", ): """Initialise instrument.""" - self.name = name - self.config = config self.input_data = input_dataset - self.kernels = kernels + self.name = input_dataset.name + self.directory = input_dataset.data_dir + self.filenames = filenames + self.variables = variables + self.dimensions = { + "lon": "longitude", + "lat": "latitude", + "time": "time", + "depth": "depth", + } # same dimensions for all instruments + self.bathymetry_file = self.directory.joinpath(bathymetry_file) + self.add_bathymetry = add_bathymetry + self.allow_time_extrapolation = allow_time_extrapolation + + def load_input_data(self) -> FieldSet: + """Load and return the input data as a FieldSet for the instrument.""" + # TODO: this should mean can delete input_data.py! + + # TODO: hopefully simulate_measurements can also be removed! And maybe the list of e.g. ctds ('measurements') to run can be added to higher level like do_expedition.py...? I think as they already do... + + # TODO: can simulate_schedule.py be refactored to be contained in base.py and repsective instrument files too...? - # def load_fieldset(self): - # """Load fieldset for simulation.""" - # # paths = self.input_data.get_fieldset_paths() - # ... + # TODO: what do I need to do about automatic registration of Instrument classes...? - def get_output_path(self, output_dir: Path) -> Path: - """Get output path for results.""" - return output_dir / f"{self.name}.zarr" + # TODO: tests will need updating...! - def run(self): + # TODO: think about combining InputDataset and Instrument classes together! + + try: + fieldset = FieldSet.from_netcdf( + self.filenames, + self.variables, + self.dimensions, + allow_time_extrapolation=self.allow_time_extrapolation, + ) + except FileNotFoundError as e: + raise FileNotFoundError( + f"Input data for instrument {self.name} not found. Have you run the `virtualship fetch` command??" + ) from e + + # interpolation methods + for var in self.variables: + getattr(fieldset, var).interp_method = "linear_invdist_land_tracer" + # depth negative + for g in fieldset.gridset.grids: + g.negate_depth() + # bathymetry data + if self.add_bathymetry: + bathymetry_field = Field.from_netcdf( + self.bathymetry_file, + self.bathymetry_variables, + self.bathymetry_dimensions, + ) + bathymetry_field.data = -bathymetry_field.data + fieldset.add_field(bathymetry_field) + fieldset.computeTimeChunk(0, 1) # read in data already + return fieldset + + @abc.abstractmethod + def simulate(self): + """Simulate instrument measurements.""" + + def run(self, *args, **kwargs): """Run instrument simulation.""" + # TODO: this will have to be able to handle the non-spinner/instead progress bar for drifters and argos! + with yaspin( text=f"Simulating {self.name} measurements... ", side="right", spinner=ship_spinner, ) as spinner: - self.simulate() + self.simulate(*args, **kwargs) spinner.ok("✅") - - @abc.abstractmethod - def simulate(self): - """Simulate instrument measurements.""" - ... diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index cd8fd330..15e81041 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -5,11 +5,11 @@ import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset +from parcels import JITParticle, ParticleSet, Variable +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType -from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.models import Spacetime +from virtualship.utils import register_input_dataset @dataclass @@ -56,7 +56,7 @@ def _ctd_cast(particle, fieldset, time): particle.delete() -@register_instrument(InstrumentType.CTD) +@register_input_dataset(InstrumentType.CTD) class CTDInputDataset(InputDataset): """Input dataset for CTD instrument.""" @@ -94,102 +94,101 @@ def get_datasets_dict(self) -> dict: } -# class CTDInstrument(instruments.Instrument): -# """CTD instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(CTD.name, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -def simulate_ctd( - fieldset: FieldSet, - out_path: str | Path, - ctds: list[CTD], - outputdt: timedelta, -) -> None: - """ - Use Parcels to simulate a set of CTDs in a fieldset. - - :param fieldset: The fieldset to simulate the CTDs in. - :param out_path: The path to write the results to. - :param ctds: A list of CTDs to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :raises ValueError: Whenever provided CTDs, fieldset, are not compatible with this function. - """ - WINCH_SPEED = 1.0 # sink and rise speed in m/s - DT = 10.0 # dt of CTD simulation integrator - - if len(ctds) == 0: - print( - "No CTDs provided. Parcels currently crashes when providing an empty particle set, so no CTD simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return - - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) +class CTDInstrument(Instrument): + """CTD instrument class.""" - # deploy time for all ctds should be later than fieldset start time - if not all( - [np.datetime64(ctd.spacetime.time) >= fieldset_starttime for ctd in ctds] + def __init__( + self, + input_dataset: InputDataset, ): - raise ValueError("CTD deployed before fieldset starts.") - - # depth the ctd will go to. shallowest between ctd max depth and bathymetry. - max_depths = [ - max( - ctd.max_depth, - fieldset.bathymetry.eval( - z=0, y=ctd.spacetime.location.lat, x=ctd.spacetime.location.lon, time=0 - ), + """Initialize CTDInstrument.""" + filenames = { + "S": input_dataset.data_dir.joinpath(f"{input_dataset.name}_s.nc"), + "T": input_dataset.data_dir.joinpath(f"{input_dataset.name}_t.nc"), + } + variables = {"S": "so", "T": "thetao"} + + super().__init__( + input_dataset, + filenames, + variables, + add_bathymetry=True, + allow_time_extrapolation=True, ) - for ctd in ctds - ] - # CTD depth can not be too shallow, because kernel would break. - # This shallow is not useful anyway, no need to support. - if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): - raise ValueError( - f"CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" + def simulate( + self, ctds: list[CTD], out_path: str | Path, outputdt: timedelta + ) -> None: + """Simulate CTD measurements.""" + WINCH_SPEED = 1.0 # sink and rise speed in m/s + DT = 10.0 # dt of CTD simulation integrator + + if len(ctds) == 0: + print( + "No CTDs provided. Parcels currently crashes when providing an empty particle set, so no CTD simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + fieldset = self.load_input_data() + + fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + + # deploy time for all ctds should be later than fieldset start time + if not all( + [np.datetime64(ctd.spacetime.time) >= fieldset_starttime for ctd in ctds] + ): + raise ValueError("CTD deployed before fieldset starts.") + + # depth the ctd will go to. shallowest between ctd max depth and bathymetry. + max_depths = [ + max( + ctd.max_depth, + fieldset.bathymetry.eval( + z=0, + y=ctd.spacetime.location.lat, + x=ctd.spacetime.location.lon, + time=0, + ), + ) + for ctd in ctds + ] + + # CTD depth can not be too shallow, because kernel would break. + # This shallow is not useful anyway, no need to support. + if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): + raise ValueError( + f"CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" + ) + + # define parcel particles + ctd_particleset = ParticleSet( + fieldset=fieldset, + pclass=_CTDParticle, + lon=[ctd.spacetime.location.lon for ctd in ctds], + lat=[ctd.spacetime.location.lat for ctd in ctds], + depth=[ctd.min_depth for ctd in ctds], + time=[ctd.spacetime.time for ctd in ctds], + max_depth=max_depths, + min_depth=[ctd.min_depth for ctd in ctds], + winch_speed=[WINCH_SPEED for _ in ctds], ) - # define parcel particles - ctd_particleset = ParticleSet( - fieldset=fieldset, - pclass=_CTDParticle, - lon=[ctd.spacetime.location.lon for ctd in ctds], - lat=[ctd.spacetime.location.lat for ctd in ctds], - depth=[ctd.min_depth for ctd in ctds], - time=[ctd.spacetime.time for ctd in ctds], - max_depth=max_depths, - min_depth=[ctd.min_depth for ctd in ctds], - winch_speed=[WINCH_SPEED for _ in ctds], - ) - - # define output file for the simulation - out_file = ctd_particleset.ParticleFile(name=out_path, outputdt=outputdt) - - # execute simulation - ctd_particleset.execute( - [_sample_salinity, _sample_temperature, _ctd_cast], - endtime=fieldset_endtime, - dt=DT, - verbose_progress=False, - output_file=out_file, - ) - - # there should be no particles left, as they delete themselves when they resurface - if len(ctd_particleset.particledata) != 0: - raise ValueError( - "Simulation ended before CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." + # define output file for the simulation + out_file = ctd_particleset.ParticleFile(name=out_path, outputdt=outputdt) + + # execute simulation + ctd_particleset.execute( + [_sample_salinity, _sample_temperature, _ctd_cast], + endtime=fieldset_endtime, + dt=DT, + verbose_progress=False, + output_file=out_file, ) + + # there should be no particles left, as they delete themselves when they resurface + if len(ctd_particleset.particledata) != 0: + raise ValueError( + "Simulation ended before CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." + ) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 92f717db..85bf02f5 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -5,11 +5,11 @@ import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset +from parcels import JITParticle, ParticleSet, Variable +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.utils import register_input_dataset, register_instrument @dataclass @@ -81,7 +81,7 @@ def _ctd_bgc_cast(particle, fieldset, time): particle.delete() -@register_instrument(InstrumentType.CTD_BGC) +@register_input_dataset(InstrumentType.CTD_BGC) class CTD_BGCInputDataset(InputDataset): """Input dataset object for CTD_BGC instrument.""" @@ -149,117 +149,128 @@ def get_datasets_dict(self) -> dict: } -# class CTD_BGCInstrument(instruments.Instrument): -# """CTD_BGC instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(CTD_BGC.name, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -def simulate_ctd_bgc( - fieldset: FieldSet, - out_path: str | Path, - ctd_bgcs: list[CTD_BGC], - outputdt: timedelta, -) -> None: - """ - Use Parcels to simulate a set of BGC CTDs in a fieldset. - - :param fieldset: The fieldset to simulate the BGC CTDs in. - :param out_path: The path to write the results to. - :param ctds: A list of BGC CTDs to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :raises ValueError: Whenever provided BGC CTDs, fieldset, are not compatible with this function. - """ - WINCH_SPEED = 1.0 # sink and rise speed in m/s - DT = 10.0 # dt of CTD simulation integrator - - if len(ctd_bgcs) == 0: - print( - "No BGC CTDs provided. Parcels currently crashes when providing an empty particle set, so no BGC CTD simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return +@register_instrument(InstrumentType.CTD_BGC) +class CTD_BGCInstrument(Instrument): + """CTD_BGC instrument class.""" - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + def __init__( + self, + input_dataset: InputDataset, + ): + """Initialize CTD_BGCInstrument.""" + filenames = { + "o2": input_dataset.data_dir.joinpath("ctd_bgc_o2.nc"), + "chl": input_dataset.data_dir.joinpath("ctd_bgc_chl.nc"), + "no3": input_dataset.data_dir.joinpath("ctd_bgc_no3.nc"), + "po4": input_dataset.data_dir.joinpath("ctd_bgc_po4.nc"), + "ph": input_dataset.data_dir.joinpath("ctd_bgc_ph.nc"), + "phyc": input_dataset.data_dir.joinpath("ctd_bgc_phyc.nc"), + "zooc": input_dataset.data_dir.joinpath("ctd_bgc_zooc.nc"), + "nppv": input_dataset.data_dir.joinpath("ctd_bgc_nppv.nc"), + } + variables = { + "o2": "o2", + "chl": "chl", + "no3": "no3", + "po4": "po4", + "ph": "ph", + "phyc": "phyc", + "zooc": "zooc", + "nppv": "nppv", + } + super().__init__( + input_dataset, + filenames, + variables, + add_bathymetry=True, + allow_time_extrapolation=True, + ) - # deploy time for all ctds should be later than fieldset start time - if not all( - [ - np.datetime64(ctd_bgc.spacetime.time) >= fieldset_starttime + def simulate( + self, ctd_bgcs: list[CTD_BGC], out_path: str | Path, outputdt: timedelta + ) -> None: + """Simulate BGC CTD measurements using Parcels.""" + WINCH_SPEED = 1.0 # sink and rise speed in m/s + DT = 10.0 # dt of CTD simulation integrator + + if len(ctd_bgcs) == 0: + print( + "No BGC CTDs provided. Parcels currently crashes when providing an empty particle set, so no BGC CTD simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + fieldset = self.load_input_data() + + fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + + # deploy time for all ctds should be later than fieldset start time + if not all( + [ + np.datetime64(ctd_bgc.spacetime.time) >= fieldset_starttime + for ctd_bgc in ctd_bgcs + ] + ): + raise ValueError("BGC CTD deployed before fieldset starts.") + + # depth the bgc ctd will go to. shallowest between bgc ctd max depth and bathymetry. + max_depths = [ + max( + ctd_bgc.max_depth, + fieldset.bathymetry.eval( + z=0, + y=ctd_bgc.spacetime.location.lat, + x=ctd_bgc.spacetime.location.lon, + time=0, + ), + ) for ctd_bgc in ctd_bgcs ] - ): - raise ValueError("BGC CTD deployed before fieldset starts.") - - # depth the bgc ctd will go to. shallowest between bgc ctd max depth and bathymetry. - max_depths = [ - max( - ctd_bgc.max_depth, - fieldset.bathymetry.eval( - z=0, - y=ctd_bgc.spacetime.location.lat, - x=ctd_bgc.spacetime.location.lon, - time=0, - ), - ) - for ctd_bgc in ctd_bgcs - ] - # CTD depth can not be too shallow, because kernel would break. - # This shallow is not useful anyway, no need to support. - if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): - raise ValueError( - f"BGC CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" + # CTD depth can not be too shallow, because kernel would break. + # This shallow is not useful anyway, no need to support. + if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): + raise ValueError( + f"BGC CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" + ) + + # define parcel particles + ctd_bgc_particleset = ParticleSet( + fieldset=fieldset, + pclass=_CTD_BGCParticle, + lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in ctd_bgcs], + lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in ctd_bgcs], + depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], + time=[ctd_bgc.spacetime.time for ctd_bgc in ctd_bgcs], + max_depth=max_depths, + min_depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], + winch_speed=[WINCH_SPEED for _ in ctd_bgcs], ) - # define parcel particles - ctd_bgc_particleset = ParticleSet( - fieldset=fieldset, - pclass=_CTD_BGCParticle, - lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in ctd_bgcs], - lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in ctd_bgcs], - depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], - time=[ctd_bgc.spacetime.time for ctd_bgc in ctd_bgcs], - max_depth=max_depths, - min_depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], - winch_speed=[WINCH_SPEED for _ in ctd_bgcs], - ) - - # define output file for the simulation - out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=outputdt) - - # execute simulation - ctd_bgc_particleset.execute( - [ - _sample_o2, - _sample_chlorophyll, - _sample_nitrate, - _sample_phosphate, - _sample_ph, - _sample_phytoplankton, - _sample_primary_production, - _ctd_bgc_cast, - ], - endtime=fieldset_endtime, - dt=DT, - verbose_progress=False, - output_file=out_file, - ) - - # there should be no particles left, as they delete themselves when they resurface - if len(ctd_bgc_particleset.particledata) != 0: - raise ValueError( - "Simulation ended before BGC CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." + # define output file for the simulation + out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=outputdt) + + # execute simulation + ctd_bgc_particleset.execute( + [ + _sample_o2, + _sample_chlorophyll, + _sample_nitrate, + _sample_phosphate, + _sample_ph, + _sample_phytoplankton, + _sample_primary_production, + _ctd_bgc_cast, + ], + endtime=fieldset_endtime, + dt=DT, + verbose_progress=False, + output_file=out_file, ) + + # there should be no particles left, as they delete themselves when they resurface + if len(ctd_bgc_particleset.particledata) != 0: + raise ValueError( + "Simulation ended before BGC CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." + ) diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 4ca0d087..186383ce 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -5,11 +5,11 @@ import numpy as np -from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset +from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.utils import register_input_dataset, register_instrument @dataclass @@ -43,7 +43,7 @@ def _check_lifetime(particle, fieldset, time): particle.delete() -@register_instrument(InstrumentType.DRIFTER) +@register_input_dataset(InstrumentType.DRIFTER) class DrifterInputDataset(InputDataset): """Input dataset for Drifter instrument.""" @@ -83,91 +83,91 @@ def get_datasets_dict(self) -> dict: } -# class DrifterInstrument(instruments.Instrument): -# """Drifter instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(Drifter.name, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -def simulate_drifters( - fieldset: FieldSet, - out_path: str | Path, - drifters: list[Drifter], - outputdt: timedelta, - dt: timedelta, - endtime: datetime | None = None, -) -> None: - """ - Use Parcels to simulate a set of drifters in a fieldset. - - :param fieldset: The fieldset to simulate the Drifters in. - :param out_path: The path to write the results to. - :param drifters: A list of drifters to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation. - :param dt: Dt for integration. - :param endtime: Stop at this time, or if None, continue until the end of the fieldset or until all drifters ended. If this is earlier than the last drifter ended or later than the end of the fieldset, a warning will be printed. - """ - if len(drifters) == 0: - print( - "No drifters provided. Parcels currently crashes when providing an empty particle set, so no drifter simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return - - # define parcel particles - drifter_particleset = ParticleSet( - fieldset=fieldset, - pclass=_DrifterParticle, - lat=[drifter.spacetime.location.lat for drifter in drifters], - lon=[drifter.spacetime.location.lon for drifter in drifters], - depth=[drifter.depth for drifter in drifters], - time=[drifter.spacetime.time for drifter in drifters], - has_lifetime=[1 if drifter.lifetime is not None else 0 for drifter in drifters], - lifetime=[ - 0 if drifter.lifetime is None else drifter.lifetime.total_seconds() - for drifter in drifters - ], - ) - - # define output file for the simulation - out_file = drifter_particleset.ParticleFile( - name=out_path, outputdt=outputdt, chunks=[len(drifter_particleset), 100] - ) - - # get earliest between fieldset end time and provide end time - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - if endtime is None: - actual_endtime = fieldset_endtime - elif endtime > fieldset_endtime: - print("WARN: Requested end time later than fieldset end time.") - actual_endtime = fieldset_endtime - else: - actual_endtime = np.timedelta64(endtime) - - # execute simulation - drifter_particleset.execute( - [AdvectionRK4, _sample_temperature, _check_lifetime], - endtime=actual_endtime, - dt=dt, - output_file=out_file, - verbose_progress=True, - ) - - # if there are more particles left than the number of drifters with an indefinite endtime, warn the user - if len(drifter_particleset.particledata) > len( - [d for d in drifters if d.lifetime is None] +@register_instrument(InstrumentType.DRIFTER) +class DrifterInstrument(Instrument): + """Drifter instrument class.""" + + def __init__( + self, + input_dataset: InputDataset, ): - print( - "WARN: Some drifters had a life time beyond the end time of the fieldset or the requested end time." + """Initialize DrifterInstrument.""" + filenames = { + "UV": input_dataset.data_dir.joinpath("drifter_uv.nc"), + "T": input_dataset.data_dir.joinpath("drifter_t.nc"), + } + variables = {"UV": ["uo", "vo"], "T": "thetao"} + super().__init__( + input_dataset, + filenames, + variables, + add_bathymetry=False, + allow_time_extrapolation=False, ) + + def simulate( + self, + drifters: list[Drifter], + out_path: str | Path, + outputdt: timedelta, + dt: timedelta, + endtime: datetime | None = None, + ) -> None: + """Simulate Drifter measurements.""" + if len(drifters) == 0: + print( + "No drifters provided. Parcels currently crashes when providing an empty particle set, so no drifter simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + fieldset = self.load_input_data() + + # define parcel particles + drifter_particleset = ParticleSet( + fieldset=fieldset, + pclass=_DrifterParticle, + lat=[drifter.spacetime.location.lat for drifter in drifters], + lon=[drifter.spacetime.location.lon for drifter in drifters], + depth=[drifter.depth for drifter in drifters], + time=[drifter.spacetime.time for drifter in drifters], + has_lifetime=[ + 1 if drifter.lifetime is not None else 0 for drifter in drifters + ], + lifetime=[ + 0 if drifter.lifetime is None else drifter.lifetime.total_seconds() + for drifter in drifters + ], + ) + + # define output file for the simulation + out_file = drifter_particleset.ParticleFile( + name=out_path, outputdt=outputdt, chunks=[len(drifter_particleset), 100] + ) + + # get earliest between fieldset end time and provide end time + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + if endtime is None: + actual_endtime = fieldset_endtime + elif endtime > fieldset_endtime: + print("WARN: Requested end time later than fieldset end time.") + actual_endtime = fieldset_endtime + else: + actual_endtime = np.timedelta64(endtime) + + # execute simulation + drifter_particleset.execute( + [AdvectionRK4, _sample_temperature, _check_lifetime], + endtime=actual_endtime, + dt=dt, + output_file=out_file, + verbose_progress=True, + ) + + # if there are more particles left than the number of drifters with an indefinite endtime, warn the user + if len(drifter_particleset.particledata) > len( + [d for d in drifters if d.lifetime is None] + ): + print( + "WARN: Some drifters had a life time beyond the end time of the fieldset or the requested end time." + ) diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index accfb5b3..371b4485 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -4,11 +4,11 @@ import numpy as np -from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.instruments.base import InputDataset +from parcels import ParticleSet, ScipyParticle, Variable +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.utils import register_input_dataset, register_instrument @dataclass @@ -36,7 +36,7 @@ def _sample_temperature(particle, fieldset, time): particle.T = fieldset.T[time, particle.depth, particle.lat, particle.lon] -@register_instrument(InstrumentType.UNDERWATER_ST) +@register_input_dataset(InstrumentType.UNDERWATER_ST) class Underwater_STInputDataset(InputDataset): """Input dataset for Underwater_ST instrument.""" @@ -76,67 +76,61 @@ def get_datasets_dict(self) -> dict: } -# class Underwater_STInstrument(instruments.Instrument): -# """Underwater_ST instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(Underwater_ST.name, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -def simulate_ship_underwater_st( - fieldset: FieldSet, - out_path: str | Path, - depth: float, - sample_points: list[Spacetime], -) -> None: - """ - Use Parcels to simulate underway data, measuring salinity and temperature at the given depth along the ship track in a fieldset. - - :param fieldset: The fieldset to simulate the sampling in. - :param out_path: The path to write the results to. - :param depth: The depth at which to measure. 0 is water surface, negative is into the water. - :param sample_points: The places and times to sample at. - """ - sample_points.sort(key=lambda p: p.time) - - particleset = ParticleSet.from_list( - fieldset=fieldset, - pclass=_ShipSTParticle, - lon=0.0, # initial lat/lon are irrelevant and will be overruled later - lat=0.0, - depth=depth, - time=0, # same for time - ) - - # define output file for the simulation - # outputdt set to infinie as we want to just want to write at the end of every call to 'execute' - out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) - - # iterate over each point, manually set lat lon time, then - # execute the particle set for one step, performing one set of measurement - for point in sample_points: - particleset.lon_nextloop[:] = point.location.lon - particleset.lat_nextloop[:] = point.location.lat - particleset.time_nextloop[:] = fieldset.time_origin.reltime( - np.datetime64(point.time) +@register_instrument(InstrumentType.UNDERWATER_ST) +class Underwater_STInstrument(Instrument): + """Underwater_ST instrument class.""" + + def __init__( + self, + input_dataset: InputDataset, + ): + """Initialize Underwater_STInstrument.""" + filenames = { + "S": input_dataset.data_dir.joinpath(f"{input_dataset.name}_s.nc"), + "T": input_dataset.data_dir.joinpath(f"{input_dataset.name}_t.nc"), + } + variables = {"S": "so", "T": "thetao"} + super().__init__( + input_dataset, + filenames, + variables, + add_bathymetry=False, + allow_time_extrapolation=True, ) - # perform one step using the particleset - # dt and runtime are set so exactly one step is made. - particleset.execute( - [_sample_salinity, _sample_temperature], - dt=1, - runtime=1, - verbose_progress=False, - output_file=out_file, + def simulate( + self, + out_path: str | Path, + depth: float, + sample_points: list[Spacetime], + ) -> None: + """Simulate underway salinity and temperature measurements.""" + sample_points.sort(key=lambda p: p.time) + + fieldset = self.load_input_data() + + particleset = ParticleSet.from_list( + fieldset=fieldset, + pclass=_ShipSTParticle, + lon=0.0, + lat=0.0, + depth=depth, + time=0, ) + + out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) + + for point in sample_points: + particleset.lon_nextloop[:] = point.location.lon + particleset.lat_nextloop[:] = point.location.lat + particleset.time_nextloop[:] = fieldset.time_origin.reltime( + np.datetime64(point.time) + ) + + particleset.execute( + [_sample_salinity, _sample_temperature], + dt=1, + runtime=1, + verbose_progress=False, + output_file=out_file, + ) diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index f5ec5fd0..c2fec98d 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -5,11 +5,11 @@ import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset +from parcels import JITParticle, ParticleSet, Variable +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.utils import register_input_dataset, register_instrument @dataclass @@ -57,7 +57,7 @@ def _xbt_cast(particle, fieldset, time): particle_ddepth = particle.max_depth - particle.depth -@register_instrument(InstrumentType.XBT) +@register_input_dataset(InstrumentType.XBT) class XBTInputDataset(InputDataset): """Input dataset for XBT instrument.""" @@ -102,106 +102,105 @@ def get_datasets_dict(self) -> dict: } -# class XBTInstrument(instruments.Instrument): -# """XBT instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(XBT.name, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -def simulate_xbt( - fieldset: FieldSet, - out_path: str | Path, - xbts: list[XBT], - outputdt: timedelta, -) -> None: - """ - Use Parcels to simulate a set of XBTs in a fieldset. - - :param fieldset: The fieldset to simulate the XBTs in. - :param out_path: The path to write the results to. - :param xbts: A list of XBTs to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :raises ValueError: Whenever provided XBTs, fieldset, are not compatible with this function. - """ - DT = 10.0 # dt of XBT simulation integrator - - if len(xbts) == 0: - print( - "No XBTs provided. Parcels currently crashes when providing an empty particle set, so no XBT simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return - - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) +@register_instrument(InstrumentType.XBT) +class XBTInstrument(Instrument): + """XBT instrument class.""" - # deploy time for all xbts should be later than fieldset start time - if not all( - [np.datetime64(xbt.spacetime.time) >= fieldset_starttime for xbt in xbts] + def __init__( + self, + input_dataset: InputDataset, ): - raise ValueError("XBT deployed before fieldset starts.") - - # depth the xbt will go to. shallowest between xbt max depth and bathymetry. - max_depths = [ - max( - xbt.max_depth, - fieldset.bathymetry.eval( - z=0, y=xbt.spacetime.location.lat, x=xbt.spacetime.location.lon, time=0 - ), + """Initialize XBTInstrument.""" + filenames = { + "UV": input_dataset.data_dir.joinpath("ship_uv.nc"), + "S": input_dataset.data_dir.joinpath("ship_s.nc"), + "T": input_dataset.data_dir.joinpath("ship_t.nc"), + } + variables = {"UV": ["uo", "vo"], "S": "so", "T": "thetao"} + super().__init__( + input_dataset, + filenames, + variables, + add_bathymetry=True, + allow_time_extrapolation=True, ) - for xbt in xbts - ] - # initial fall speeds - initial_fall_speeds = [xbt.fall_speed for xbt in xbts] - - # XBT depth can not be too shallow, because kernel would break. - # This shallow is not useful anyway, no need to support. - # TODO: should this be more informative? Is "maximum" right? Should tell user can't use XBT here? - for max_depth, fall_speed in zip(max_depths, initial_fall_speeds, strict=False): - if not max_depth <= -DT * fall_speed: - raise ValueError( - f"XBT max_depth or bathymetry shallower than maximum {-DT * fall_speed}" + def simulate( + self, + xbts: list[XBT], + out_path: str | Path, + outputdt: timedelta, + ) -> None: + """Simulate XBT measurements.""" + DT = 10.0 # dt of XBT simulation integrator + + if len(xbts) == 0: + print( + "No XBTs provided. Parcels currently crashes when providing an empty particle set, so no XBT simulation will be done and no files will be created." ) + # TODO when Parcels supports it this check can be removed. + return + + fieldset = self.load_input_data() + + fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + + # deploy time for all xbts should be later than fieldset start time + if not all( + [np.datetime64(xbt.spacetime.time) >= fieldset_starttime for xbt in xbts] + ): + raise ValueError("XBT deployed before fieldset starts.") + + # depth the xbt will go to. shallowest between xbt max depth and bathymetry. + max_depths = [ + max( + xbt.max_depth, + fieldset.bathymetry.eval( + z=0, + y=xbt.spacetime.location.lat, + x=xbt.spacetime.location.lon, + time=0, + ), + ) + for xbt in xbts + ] + + # initial fall speeds + initial_fall_speeds = [xbt.fall_speed for xbt in xbts] + + # XBT depth can not be too shallow, because kernel would break. + for max_depth, fall_speed in zip(max_depths, initial_fall_speeds, strict=False): + if not max_depth <= -DT * fall_speed: + raise ValueError( + f"XBT max_depth or bathymetry shallower than maximum {-DT * fall_speed}" + ) + + # define xbt particles + xbt_particleset = ParticleSet( + fieldset=fieldset, + pclass=_XBTParticle, + lon=[xbt.spacetime.location.lon for xbt in xbts], + lat=[xbt.spacetime.location.lat for xbt in xbts], + depth=[xbt.min_depth for xbt in xbts], + time=[xbt.spacetime.time for xbt in xbts], + max_depth=max_depths, + min_depth=[xbt.min_depth for xbt in xbts], + fall_speed=[xbt.fall_speed for xbt in xbts], + ) - # define xbt particles - xbt_particleset = ParticleSet( - fieldset=fieldset, - pclass=_XBTParticle, - lon=[xbt.spacetime.location.lon for xbt in xbts], - lat=[xbt.spacetime.location.lat for xbt in xbts], - depth=[xbt.min_depth for xbt in xbts], - time=[xbt.spacetime.time for xbt in xbts], - max_depth=max_depths, - min_depth=[xbt.min_depth for xbt in xbts], - fall_speed=[xbt.fall_speed for xbt in xbts], - ) - - # define output file for the simulation - out_file = xbt_particleset.ParticleFile(name=out_path, outputdt=outputdt) + out_file = xbt_particleset.ParticleFile(name=out_path, outputdt=outputdt) - # execute simulation - xbt_particleset.execute( - [_sample_temperature, _xbt_cast], - endtime=fieldset_endtime, - dt=DT, - verbose_progress=False, - output_file=out_file, - ) - - # there should be no particles left, as they delete themselves when they finish profiling - if len(xbt_particleset.particledata) != 0: - raise ValueError( - "Simulation ended before XBT finished profiling. This most likely means the field time dimension did not match the simulation time span." + xbt_particleset.execute( + [_sample_temperature, _xbt_cast], + endtime=fieldset_endtime, + dt=DT, + verbose_progress=False, + output_file=out_file, ) + + # there should be no particles left, as they delete themselves when they finish profiling + if len(xbt_particleset.particledata) != 0: + raise ValueError( + "Simulation ended before XBT finished profiling. This most likely means the field time dimension did not match the simulation time span." + ) diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index d7559fd0..9add2c1b 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -17,7 +17,6 @@ if TYPE_CHECKING: from parcels import FieldSet - from virtualship.expedition.input_data import InputData projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") @@ -88,7 +87,6 @@ class Schedule(pydantic.BaseModel): def verify( self, ship_speed: float, - input_data: InputData | None, *, check_space_time_region: bool = False, ignore_missing_fieldsets: bool = False, @@ -139,19 +137,20 @@ def verify( # check if all waypoints are in water # this is done by picking an arbitrary provided fieldset and checking if UV is not zero + # TODO: this may need to be done with generic bathymetry data, now that removed InputData! # get all available fieldsets available_fieldsets = [] - if input_data is not None: - fieldsets = [ - input_data.adcp_fieldset, - input_data.argo_float_fieldset, - input_data.ctd_fieldset, - input_data.drifter_fieldset, - input_data.ship_underwater_st_fieldset, - ] - for fs in fieldsets: - if fs is not None: - available_fieldsets.append(fs) + # if input_data is not None: + # fieldsets = [ + # input_data.adcp_fieldset, + # input_data.argo_float_fieldset, + # input_data.ctd_fieldset, + # input_data.drifter_fieldset, + # input_data.ship_underwater_st_fieldset, + # ] + # for fs in fieldsets: + # if fs is not None: + # available_fieldsets.append(fs) # check if there are any fieldsets, else it's an error if len(available_fieldsets) == 0: diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index e1054da1..6ffaefe7 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -240,17 +240,31 @@ def _get_expedition(expedition_dir: Path) -> Expedition: ], ) -# InstrumentType -> InputDataset registry and registration utilities. -INSTRUMENT_INPUT_DATASET_MAP = {} + +# InstrumentType -> InputDataset and Instrument registry and registration utilities. +INPUT_DATASET_MAP = {} +INSTRUMENT_CLASS_MAP = {} + + +def register_input_dataset(instrument_type): + def decorator(cls): + INPUT_DATASET_MAP[instrument_type] = cls + return cls + + return decorator def register_instrument(instrument_type): def decorator(cls): - INSTRUMENT_INPUT_DATASET_MAP[instrument_type] = cls + INSTRUMENT_CLASS_MAP[instrument_type] = cls return cls return decorator def get_input_dataset_class(instrument_type): - return INSTRUMENT_INPUT_DATASET_MAP.get(instrument_type) + return INPUT_DATASET_MAP.get(instrument_type) + + +def get_instrument_class(instrument_type): + return INSTRUMENT_CLASS_MAP.get(instrument_type) From e96aa8a3775eed9cc45fea5553a7a66fa461aeaf Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 28 Oct 2025 13:45:19 +0100 Subject: [PATCH 29/97] evaporate simulate_measurements.py; centralise run logic --- src/virtualship/expedition/do_expedition.py | 57 +++--- .../expedition/simulate_measurements.py | 162 ------------------ src/virtualship/instruments/adcp.py | 39 ++--- src/virtualship/instruments/argo_float.py | 66 ++++--- src/virtualship/instruments/base.py | 21 ++- src/virtualship/instruments/ctd.py | 42 ++--- src/virtualship/instruments/ctd_bgc.py | 71 ++++---- src/virtualship/instruments/drifter.py | 62 ++++--- .../instruments/ship_underwater_st.py | 32 ++-- src/virtualship/instruments/xbt.py | 55 +++--- 10 files changed, 211 insertions(+), 396 deletions(-) delete mode 100644 src/virtualship/expedition/simulate_measurements.py diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index ef5b6037..c4bc6783 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -44,17 +44,12 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> # verify that schedule and checkpoint match checkpoint.verify(expedition.schedule) - # load fieldsets - _load_input_data = [] # TEMPORARY! - loaded_input_data = _load_input_data( - expedition_dir=expedition_dir, - expedition=expedition, - input_data=input_data, - ) - print("\n---- WAYPOINT VERIFICATION ----") # verify schedule is valid + # TODO: needs updating when .verify() updated to not need input_data + + loaded_input_data = [] # TODO: TEMPORARY! expedition.schedule.verify( expedition.ship_config.ship_speed_knots, loaded_input_data ) @@ -87,48 +82,34 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> print("\n----- EXPEDITION SUMMARY ------") - # calculate expedition cost in US$ - assert expedition.schedule.waypoints[0].time is not None, ( - "First waypoint has no time. This should not be possible as it should have been verified before." - ) - time_past = schedule_results.time - expedition.schedule.waypoints[0].time - cost = expedition_cost(schedule_results, time_past) - with open(expedition_dir.joinpath("results", "cost.txt"), "w") as file: - file.writelines(f"cost: {cost} US$") - print(f"\nExpedition duration: {time_past}\nExpedition cost: US$ {cost:,.0f}.") + # expedition cost in US$ + _write_expedition_cost(expedition, schedule_results, expedition_dir) print("\n--- MEASUREMENT SIMULATIONS ---") # simulate measurements print("\nSimulating measurements. This may take a while...\n") - # TODO: this is where XYZInstrument.run() could be called instead of simulate_measurements!? - # TODO: this time maybe looping through measurements to simulate in some form... - # TODO: first in explicit per instrument, then think about whether can be automated more...not the end of the world if just have to explain in documentation that changes must be made here... - instruments_in_expedition = expedition.get_instruments() for itype in instruments_in_expedition: + # get instrument class instrument_class = get_instrument_class(itype) if instrument_class is None: raise RuntimeError(f"No instrument class found for type {itype}.") + # get measurements to simulate for this instrument measurements = schedule_results.measurements_to_simulate.get(itype.name.lower()) - instrument_class.run( - expedition_dir.joinpath("results", f"{itype.name.lower()}.zarr"), + # initialise instrument + instrument = instrument_class(expedition=expedition, directory=expedition_dir) + + # run simulation + instrument.run( measurements=measurements, - fieldset=loaded_input_data.get_fieldset_for_instrument_type(itype), - expedition=expedition, + out_path=expedition_dir.joinpath("results", f"{itype.name.lower()}.zarr"), ) - # simulate_measurements( - # expedition_dir, - # expedition.instruments_config, - # loaded_input_data, - # schedule_results.measurements_to_simulate, - # ) - print("\nAll measurement simulations are complete.") print("\n----- EXPEDITION RESULTS ------") @@ -150,3 +131,15 @@ def _load_checkpoint(expedition_dir: Path) -> Checkpoint | None: def _save_checkpoint(checkpoint: Checkpoint, expedition_dir: Path) -> None: file_path = expedition_dir.joinpath(CHECKPOINT) checkpoint.to_yaml(file_path) + + +def _write_expedition_cost(expedition, schedule_results, expedition_dir): + """Calculate the expedition cost, write it to a file, and print summary.""" + assert expedition.schedule.waypoints[0].time is not None, ( + "First waypoint has no time. This should not be possible as it should have been verified before." + ) + time_past = schedule_results.time - expedition.schedule.waypoints[0].time + cost = expedition_cost(schedule_results, time_past) + with open(expedition_dir.joinpath("results", "cost.txt"), "w") as file: + file.writelines(f"cost: {cost} US$") + print(f"\nExpedition duration: {time_past}\nExpedition cost: US$ {cost:,.0f}.") diff --git a/src/virtualship/expedition/simulate_measurements.py b/src/virtualship/expedition/simulate_measurements.py deleted file mode 100644 index 6cb2e488..00000000 --- a/src/virtualship/expedition/simulate_measurements.py +++ /dev/null @@ -1,162 +0,0 @@ -"""simulate_measurements function.""" - -from __future__ import annotations - -import logging -from datetime import timedelta -from pathlib import Path -from typing import TYPE_CHECKING - -from yaspin import yaspin - -from virtualship.instruments.adcp import simulate_adcp -from virtualship.instruments.argo_float import simulate_argo_floats -from virtualship.instruments.ctd import simulate_ctd -from virtualship.instruments.ctd_bgc import simulate_ctd_bgc -from virtualship.instruments.drifter import simulate_drifters -from virtualship.instruments.ship_underwater_st import simulate_ship_underwater_st -from virtualship.instruments.xbt import simulate_xbt -from virtualship.models import InstrumentsConfig -from virtualship.utils import ship_spinner - -from .simulate_schedule import MeasurementsToSimulate - -if TYPE_CHECKING: - from .input_data import InputData - -# parcels logger (suppress INFO messages to prevent log being flooded) -external_logger = logging.getLogger("parcels.tools.loggers") -external_logger.setLevel(logging.WARNING) - - -def simulate_measurements( - expedition_dir: str | Path, - instruments_config: InstrumentsConfig, - input_data: InputData, - measurements: MeasurementsToSimulate, -) -> None: - """ - Simulate measurements using Parcels. - - Saves everything in expedition_dir/results. - - :param expedition_dir: Base directory of the expedition. - :param input_data: Input data for simulation. - :param measurements: The measurements to simulate. - :raises RuntimeError: In case fieldsets of configuration is not provided. Make sure to check this before calling this function. - """ - if isinstance(expedition_dir, str): - expedition_dir = Path(expedition_dir) - - if len(measurements.ship_underwater_sts) > 0: - if instruments_config.ship_underwater_st_config is None: - raise RuntimeError("No configuration for ship underwater ST provided.") - if input_data.ship_underwater_st_fieldset is None: - raise RuntimeError("No fieldset for ship underwater ST provided.") - with yaspin( - text="Simulating onboard temperature and salinity measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: - simulate_ship_underwater_st( - fieldset=input_data.ship_underwater_st_fieldset, - out_path=expedition_dir.joinpath("results", "ship_underwater_st.zarr"), - depth=-2, - sample_points=measurements.ship_underwater_sts, - ) - spinner.ok("✅") - - if len(measurements.adcps) > 0: - if instruments_config.adcp_config is None: - raise RuntimeError("No configuration for ADCP provided.") - if input_data.adcp_fieldset is None: - raise RuntimeError("No fieldset for ADCP provided.") - with yaspin( - text="Simulating onboard ADCP... ", side="right", spinner=ship_spinner - ) as spinner: - simulate_adcp( - fieldset=input_data.adcp_fieldset, - out_path=expedition_dir.joinpath("results", "adcp.zarr"), - max_depth=instruments_config.adcp_config.max_depth_meter, - min_depth=-5, - num_bins=instruments_config.adcp_config.num_bins, - sample_points=measurements.adcps, - ) - spinner.ok("✅") - - if len(measurements.ctds) > 0: - if instruments_config.ctd_config is None: - raise RuntimeError("No configuration for CTD provided.") - if input_data.ctd_fieldset is None: - raise RuntimeError("No fieldset for CTD provided.") - with yaspin( - text="Simulating CTD casts... ", side="right", spinner=ship_spinner - ) as spinner: - simulate_ctd( - out_path=expedition_dir.joinpath("results", "ctd.zarr"), - fieldset=input_data.ctd_fieldset, - ctds=measurements.ctds, - outputdt=timedelta(seconds=10), - ) - spinner.ok("✅") - - if len(measurements.ctd_bgcs) > 0: - if instruments_config.ctd_bgc_config is None: - raise RuntimeError("No configuration for CTD_BGC provided.") - if input_data.ctd_bgc_fieldset is None: - raise RuntimeError("No fieldset for CTD_BGC provided.") - with yaspin( - text="Simulating BGC CTD casts... ", side="right", spinner=ship_spinner - ) as spinner: - simulate_ctd_bgc( - out_path=expedition_dir.joinpath("results", "ctd_bgc.zarr"), - fieldset=input_data.ctd_bgc_fieldset, - ctd_bgcs=measurements.ctd_bgcs, - outputdt=timedelta(seconds=10), - ) - spinner.ok("✅") - - if len(measurements.xbts) > 0: - if instruments_config.xbt_config is None: - raise RuntimeError("No configuration for XBTs provided.") - if input_data.xbt_fieldset is None: - raise RuntimeError("No fieldset for XBTs provided.") - with yaspin( - text="Simulating XBTs... ", side="right", spinner=ship_spinner - ) as spinner: - simulate_xbt( - out_path=expedition_dir.joinpath("results", "xbts.zarr"), - fieldset=input_data.xbt_fieldset, - xbts=measurements.xbts, - outputdt=timedelta(seconds=1), - ) - spinner.ok("✅") - - if len(measurements.drifters) > 0: - print("Simulating drifters... ") - if instruments_config.drifter_config is None: - raise RuntimeError("No configuration for drifters provided.") - if input_data.drifter_fieldset is None: - raise RuntimeError("No fieldset for drifters provided.") - simulate_drifters( - out_path=expedition_dir.joinpath("results", "drifters.zarr"), - fieldset=input_data.drifter_fieldset, - drifters=measurements.drifters, - outputdt=timedelta(hours=5), - dt=timedelta(minutes=5), - endtime=None, - ) - - if len(measurements.argo_floats) > 0: - print("Simulating argo floats... ") - if instruments_config.argo_float_config is None: - raise RuntimeError("No configuration for argo floats provided.") - if input_data.argo_float_fieldset is None: - raise RuntimeError("No fieldset for argo floats provided.") - simulate_argo_floats( - out_path=expedition_dir.joinpath("results", "argo_floats.zarr"), - argo_floats=measurements.argo_floats, - fieldset=input_data.argo_float_fieldset, - outputdt=timedelta(minutes=5), - endtime=None, - ) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 5caab5bf..917da154 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from pathlib import Path from typing import ClassVar import numpy as np @@ -7,8 +6,10 @@ from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType -from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_input_dataset, register_instrument +from virtualship.utils import ( + register_input_dataset, + register_instrument, +) @dataclass @@ -73,37 +74,33 @@ def get_datasets_dict(self) -> dict: class ADCPInstrument(Instrument): """ADCP instrument class.""" - def __init__( - self, - input_dataset: InputDataset, - ): + def __init__(self, name, expedition, directory): """Initialize ADCPInstrument.""" filenames = { - "UV": input_dataset.data_dir.joinpath(f"{input_dataset.name}_uv.nc"), + "UV": directory.joinpath(f"{name}_uv.nc"), } variables = {"UV": ["uo", "vo"]} super().__init__( - input_dataset, + ADCP.name, + expedition, + directory, filenames, variables, add_bathymetry=False, allow_time_extrapolation=True, ) - def simulate( - self, - out_path: str | Path, - max_depth: float, - min_depth: float, - num_bins: int, - sample_points: list[Spacetime], - ) -> None: + def simulate(self) -> None: """Simulate ADCP measurements.""" - sample_points.sort(key=lambda p: p.time) + MAX_DEPTH = self.expedition.instruments_config.adcp_config.max_depth_meter + MIN_DEPTH = -5.0 + NUM_BINS = self.instruments_config.adcp_config.num_bins + + self.measurements.sort(key=lambda p: p.time) fieldset = self.load_input_data() - bins = np.linspace(max_depth, min_depth, num_bins) + bins = np.linspace(MAX_DEPTH, MIN_DEPTH, NUM_BINS) num_particles = len(bins) particleset = ParticleSet.from_list( fieldset=fieldset, @@ -114,9 +111,9 @@ def simulate( time=0, ) - out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) + out_file = particleset.ParticleFile(name=self.out_path, outputdt=np.inf) - for point in sample_points: + for point in self.measurements: particleset.lon_nextloop[:] = point.location.lon particleset.lat_nextloop[:] = point.location.lat particleset.time_nextloop[:] = fieldset.time_origin.reltime( diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 66b25bdf..239b47de 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -1,7 +1,6 @@ import math from dataclasses import dataclass -from datetime import datetime, timedelta -from pathlib import Path +from datetime import timedelta from typing import ClassVar import numpy as np @@ -147,17 +146,17 @@ def get_datasets_dict(self) -> dict: "UVdata": { "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", "variables": ["uo", "vo"], - "output_filename": "argo_float_uv.nc", + "output_filename": f"{self.name}_uv.nc", }, "Sdata": { "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", "variables": ["so"], - "output_filename": "argo_float_s.nc", + "output_filename": f"{self.name}_s.nc", }, "Tdata": { "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", "variables": ["thetao"], - "output_filename": "argo_float_t.nc", + "output_filename": f"{self.name}_t.nc", }, } @@ -166,36 +165,31 @@ def get_datasets_dict(self) -> dict: class ArgoFloatInstrument(Instrument): """ArgoFloat instrument class.""" - def __init__( - self, - input_dataset: InputDataset, - ): + def __init__(self, name, expedition, directory): """Initialize ArgoFloatInstrument.""" filenames = { - "UV": input_dataset.data_dir.joinpath("argo_float_uv.nc"), - "S": input_dataset.data_dir.joinpath("argo_float_s.nc"), - "T": input_dataset.data_dir.joinpath("argo_float_t.nc"), + "UV": directory.joinpath(f"{name}_uv.nc"), + "S": directory.joinpath(f"{name}_s.nc"), + "T": directory.joinpath(f"{name}_t.nc"), } variables = {"UV": ["uo", "vo"], "S": "so", "T": "thetao"} super().__init__( - input_dataset, + ArgoFloat.name, + expedition, + directory, filenames, variables, add_bathymetry=False, allow_time_extrapolation=False, ) - def simulate( - self, - argo_floats: list[ArgoFloat], - out_path: str | Path, - outputdt: timedelta, - endtime: datetime | None = None, - ) -> None: + def simulate(self) -> None: """Simulate Argo float measurements.""" DT = 10.0 # dt of Argo float simulation integrator + OUTPUT_DT = timedelta(minutes=5) + ENDTIME = None - if len(argo_floats) == 0: + if len(self.measurements) == 0: print( "No Argo floats provided. Parcels currently crashes when providing an empty particle set, so no argo floats simulation will be done and no files will be created." ) @@ -208,32 +202,34 @@ def simulate( argo_float_particleset = ParticleSet( fieldset=fieldset, pclass=_ArgoParticle, - lat=[argo.spacetime.location.lat for argo in argo_floats], - lon=[argo.spacetime.location.lon for argo in argo_floats], - depth=[argo.min_depth for argo in argo_floats], - time=[argo.spacetime.time for argo in argo_floats], - min_depth=[argo.min_depth for argo in argo_floats], - max_depth=[argo.max_depth for argo in argo_floats], - drift_depth=[argo.drift_depth for argo in argo_floats], - vertical_speed=[argo.vertical_speed for argo in argo_floats], - cycle_days=[argo.cycle_days for argo in argo_floats], - drift_days=[argo.drift_days for argo in argo_floats], + lat=[argo.spacetime.location.lat for argo in self.measurements], + lon=[argo.spacetime.location.lon for argo in self.measurements], + depth=[argo.min_depth for argo in self.measurements], + time=[argo.spacetime.time for argo in self.measurements], + min_depth=[argo.min_depth for argo in self.measurements], + max_depth=[argo.max_depth for argo in self.measurements], + drift_depth=[argo.drift_depth for argo in self.measurements], + vertical_speed=[argo.vertical_speed for argo in self.measurements], + cycle_days=[argo.cycle_days for argo in self.measurements], + drift_days=[argo.drift_days for argo in self.measurements], ) # define output file for the simulation out_file = argo_float_particleset.ParticleFile( - name=out_path, outputdt=outputdt, chunks=[len(argo_float_particleset), 100] + name=self.out_path, + outputdt=OUTPUT_DT, + chunks=[len(argo_float_particleset), 100], ) # get earliest between fieldset end time and provide end time fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - if endtime is None: + if ENDTIME is None: actual_endtime = fieldset_endtime - elif endtime > fieldset_endtime: + elif ENDTIME > fieldset_endtime: print("WARN: Requested end time later than fieldset end time.") actual_endtime = fieldset_endtime else: - actual_endtime = np.timedelta64(endtime) + actual_endtime = np.timedelta64(ENDTIME) # execute simulation argo_float_particleset.execute( diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 2fab4795..13a2fb05 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,11 +1,12 @@ import abc from datetime import timedelta +from pathlib import Path import copernicusmarine import yaspin from parcels import Field, FieldSet -from virtualship.models import SpaceTimeRegion +from virtualship.models import Expedition, SpaceTimeRegion from virtualship.utils import ship_spinner @@ -71,7 +72,9 @@ class Instrument(abc.ABC): def __init__( self, - input_dataset: InputDataset, + name: str, + expedition: Expedition, + directory: Path | str, filenames: dict, variables: dict, add_bathymetry: bool, @@ -79,9 +82,9 @@ def __init__( bathymetry_file: str = "bathymetry.nc", ): """Initialise instrument.""" - self.input_data = input_dataset - self.name = input_dataset.name - self.directory = input_dataset.data_dir + self.name = name + self.expedition = expedition + self.directory = directory self.filenames = filenames self.variables = variables self.dimensions = { @@ -106,7 +109,7 @@ def load_input_data(self) -> FieldSet: # TODO: tests will need updating...! - # TODO: think about combining InputDataset and Instrument classes together! + # TODO: think about combining InputDataset and Instrument classes together! Or maybe not if they are better kept separate... try: fieldset = FieldSet.from_netcdf( @@ -139,10 +142,10 @@ def load_input_data(self) -> FieldSet: return fieldset @abc.abstractmethod - def simulate(self): + def simulate(self, measurements: list, out_path: str | Path): """Simulate instrument measurements.""" - def run(self, *args, **kwargs): + def run(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" # TODO: this will have to be able to handle the non-spinner/instead progress bar for drifters and argos! @@ -151,5 +154,5 @@ def run(self, *args, **kwargs): side="right", spinner=ship_spinner, ) as spinner: - self.simulate(*args, **kwargs) + self.simulate(measurements, out_path) spinner.ok("✅") diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 15e81041..82cace94 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -1,6 +1,5 @@ from dataclasses import dataclass from datetime import timedelta -from pathlib import Path from typing import ClassVar import numpy as np @@ -97,33 +96,31 @@ def get_datasets_dict(self) -> dict: class CTDInstrument(Instrument): """CTD instrument class.""" - def __init__( - self, - input_dataset: InputDataset, - ): + def __init__(self, name, expedition, directory): """Initialize CTDInstrument.""" filenames = { - "S": input_dataset.data_dir.joinpath(f"{input_dataset.name}_s.nc"), - "T": input_dataset.data_dir.joinpath(f"{input_dataset.name}_t.nc"), + "S": directory.data_dir.joinpath(f"{name}_s.nc"), + "T": directory.data_dir.joinpath(f"{name}_t.nc"), } variables = {"S": "so", "T": "thetao"} super().__init__( - input_dataset, + CTD.name, + expedition, + directory, filenames, variables, add_bathymetry=True, allow_time_extrapolation=True, ) - def simulate( - self, ctds: list[CTD], out_path: str | Path, outputdt: timedelta - ) -> None: + def simulate(self) -> None: """Simulate CTD measurements.""" WINCH_SPEED = 1.0 # sink and rise speed in m/s DT = 10.0 # dt of CTD simulation integrator + OUTPUT_DT = timedelta(seconds=10) # output dt for CTD simulation - if len(ctds) == 0: + if len(self.measurements) == 0: print( "No CTDs provided. Parcels currently crashes when providing an empty particle set, so no CTD simulation will be done and no files will be created." ) @@ -137,7 +134,10 @@ def simulate( # deploy time for all ctds should be later than fieldset start time if not all( - [np.datetime64(ctd.spacetime.time) >= fieldset_starttime for ctd in ctds] + [ + np.datetime64(ctd.spacetime.time) >= fieldset_starttime + for ctd in self.measurements + ] ): raise ValueError("CTD deployed before fieldset starts.") @@ -152,7 +152,7 @@ def simulate( time=0, ), ) - for ctd in ctds + for ctd in self.measurements ] # CTD depth can not be too shallow, because kernel would break. @@ -166,17 +166,17 @@ def simulate( ctd_particleset = ParticleSet( fieldset=fieldset, pclass=_CTDParticle, - lon=[ctd.spacetime.location.lon for ctd in ctds], - lat=[ctd.spacetime.location.lat for ctd in ctds], - depth=[ctd.min_depth for ctd in ctds], - time=[ctd.spacetime.time for ctd in ctds], + lon=[ctd.spacetime.location.lon for ctd in self.measurements], + lat=[ctd.spacetime.location.lat for ctd in self.measurements], + depth=[ctd.min_depth for ctd in self.measurements], + time=[ctd.spacetime.time for ctd in self.measurements], max_depth=max_depths, - min_depth=[ctd.min_depth for ctd in ctds], - winch_speed=[WINCH_SPEED for _ in ctds], + min_depth=[ctd.min_depth for ctd in self.measurements], + winch_speed=[WINCH_SPEED for _ in self.measurements], ) # define output file for the simulation - out_file = ctd_particleset.ParticleFile(name=out_path, outputdt=outputdt) + out_file = ctd_particleset.ParticleFile(name=self.out_path, outputdt=OUTPUT_DT) # execute simulation ctd_particleset.execute( diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 85bf02f5..b017e0f4 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -1,6 +1,5 @@ from dataclasses import dataclass from datetime import timedelta -from pathlib import Path from typing import ClassVar import numpy as np @@ -109,42 +108,42 @@ def get_datasets_dict(self) -> dict: "o2data": { "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", "variables": ["o2"], - "output_filename": "ctd_bgc_o2.nc", + "output_filename": f"{self.name}_o2.nc", }, "chlorodata": { "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", "variables": ["chl"], - "output_filename": "ctd_bgc_chl.nc", + "output_filename": f"{self.name}_chl.nc", }, "nitratedata": { "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", "variables": ["no3"], - "output_filename": "ctd_bgc_no3.nc", + "output_filename": f"{self.name}_no3.nc", }, "phosphatedata": { "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", "variables": ["po4"], - "output_filename": "ctd_bgc_po4.nc", + "output_filename": f"{self.name}_po4.nc", }, "phdata": { "dataset_id": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", "variables": ["ph"], - "output_filename": "ctd_bgc_ph.nc", + "output_filename": f"{self.name}_ph.nc", }, "phytoplanktondata": { "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", "variables": ["phyc"], - "output_filename": "ctd_bgc_phyc.nc", + "output_filename": f"{self.name}_phyc.nc", }, "zooplanktondata": { "dataset_id": "cmems_mod_glo_bgc-plankton_anfc_0.25deg_P1D-m", "variables": ["zooc"], - "output_filename": "ctd_bgc_zooc.nc", + "output_filename": f"{self.name}_zooc.nc", }, "primaryproductiondata": { "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", "variables": ["nppv"], - "output_filename": "ctd_bgc_nppv.nc", + "output_filename": f"{self.name}_nppv.nc", }, } @@ -153,20 +152,17 @@ def get_datasets_dict(self) -> dict: class CTD_BGCInstrument(Instrument): """CTD_BGC instrument class.""" - def __init__( - self, - input_dataset: InputDataset, - ): + def __init__(self, name, expedition, directory): """Initialize CTD_BGCInstrument.""" filenames = { - "o2": input_dataset.data_dir.joinpath("ctd_bgc_o2.nc"), - "chl": input_dataset.data_dir.joinpath("ctd_bgc_chl.nc"), - "no3": input_dataset.data_dir.joinpath("ctd_bgc_no3.nc"), - "po4": input_dataset.data_dir.joinpath("ctd_bgc_po4.nc"), - "ph": input_dataset.data_dir.joinpath("ctd_bgc_ph.nc"), - "phyc": input_dataset.data_dir.joinpath("ctd_bgc_phyc.nc"), - "zooc": input_dataset.data_dir.joinpath("ctd_bgc_zooc.nc"), - "nppv": input_dataset.data_dir.joinpath("ctd_bgc_nppv.nc"), + "o2": directory.joinpath(f"{name}_o2.nc"), + "chl": directory.joinpath(f"{name}_chl.nc"), + "no3": directory.joinpath(f"{name}_no3.nc"), + "po4": directory.joinpath(f"{name}_po4.nc"), + "ph": directory.joinpath(f"{name}_ph.nc"), + "phyc": directory.joinpath(f"{name}_phyc.nc"), + "zooc": directory.joinpath(f"{name}_zooc.nc"), + "nppv": directory.joinpath(f"{name}_nppv.nc"), } variables = { "o2": "o2", @@ -179,21 +175,22 @@ def __init__( "nppv": "nppv", } super().__init__( - input_dataset, + CTD_BGC.name, + expedition, + directory, filenames, variables, add_bathymetry=True, allow_time_extrapolation=True, ) - def simulate( - self, ctd_bgcs: list[CTD_BGC], out_path: str | Path, outputdt: timedelta - ) -> None: + def simulate(self) -> None: """Simulate BGC CTD measurements using Parcels.""" WINCH_SPEED = 1.0 # sink and rise speed in m/s - DT = 10.0 # dt of CTD simulation integrator + DT = 10.0 # dt of CTD_BGC simulation integrator + OUTPUT_DT = timedelta(seconds=10) # output dt for CTD_BGC simulation - if len(ctd_bgcs) == 0: + if len(self.measurements) == 0: print( "No BGC CTDs provided. Parcels currently crashes when providing an empty particle set, so no BGC CTD simulation will be done and no files will be created." ) @@ -209,7 +206,7 @@ def simulate( if not all( [ np.datetime64(ctd_bgc.spacetime.time) >= fieldset_starttime - for ctd_bgc in ctd_bgcs + for ctd_bgc in self.measurements ] ): raise ValueError("BGC CTD deployed before fieldset starts.") @@ -225,7 +222,7 @@ def simulate( time=0, ), ) - for ctd_bgc in ctd_bgcs + for ctd_bgc in self.measurements ] # CTD depth can not be too shallow, because kernel would break. @@ -239,17 +236,19 @@ def simulate( ctd_bgc_particleset = ParticleSet( fieldset=fieldset, pclass=_CTD_BGCParticle, - lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in ctd_bgcs], - lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in ctd_bgcs], - depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], - time=[ctd_bgc.spacetime.time for ctd_bgc in ctd_bgcs], + lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in self.measurements], + lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in self.measurements], + depth=[ctd_bgc.min_depth for ctd_bgc in self.measurements], + time=[ctd_bgc.spacetime.time for ctd_bgc in self.measurements], max_depth=max_depths, - min_depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], - winch_speed=[WINCH_SPEED for _ in ctd_bgcs], + min_depth=[ctd_bgc.min_depth for ctd_bgc in self.measurements], + winch_speed=[WINCH_SPEED for _ in self.measurements], ) # define output file for the simulation - out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=outputdt) + out_file = ctd_bgc_particleset.ParticleFile( + name=self.out_path, outputdt=OUTPUT_DT + ) # execute simulation ctd_bgc_particleset.execute( diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 186383ce..72db065e 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -1,6 +1,5 @@ from dataclasses import dataclass -from datetime import datetime, timedelta -from pathlib import Path +from datetime import timedelta from typing import ClassVar import numpy as np @@ -73,12 +72,12 @@ def get_datasets_dict(self) -> dict: "UVdata": { "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", "variables": ["uo", "vo"], - "output_filename": "drifter_uv.nc", + "output_filename": f"{self.name}_uv.nc", }, "Tdata": { "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", "variables": ["thetao"], - "output_filename": "drifter_t.nc", + "output_filename": f"{self.name}_t.nc", }, } @@ -87,34 +86,30 @@ def get_datasets_dict(self) -> dict: class DrifterInstrument(Instrument): """Drifter instrument class.""" - def __init__( - self, - input_dataset: InputDataset, - ): + def __init__(self, name, expedition, directory): """Initialize DrifterInstrument.""" filenames = { - "UV": input_dataset.data_dir.joinpath("drifter_uv.nc"), - "T": input_dataset.data_dir.joinpath("drifter_t.nc"), + "UV": directory.joinpath(f"{name}_uv.nc"), + "T": directory.joinpath(f"{name}_t.nc"), } variables = {"UV": ["uo", "vo"], "T": "thetao"} super().__init__( - input_dataset, + Drifter.name, + expedition, + directory, filenames, variables, add_bathymetry=False, allow_time_extrapolation=False, ) - def simulate( - self, - drifters: list[Drifter], - out_path: str | Path, - outputdt: timedelta, - dt: timedelta, - endtime: datetime | None = None, - ) -> None: + def simulate(self) -> None: """Simulate Drifter measurements.""" - if len(drifters) == 0: + OUTPUT_DT = timedelta(hours=5) + DT = timedelta(minutes=5) + ENDTIME = None + + if len(self.measurements) == 0: print( "No drifters provided. Parcels currently crashes when providing an empty particle set, so no drifter simulation will be done and no files will be created." ) @@ -127,46 +122,49 @@ def simulate( drifter_particleset = ParticleSet( fieldset=fieldset, pclass=_DrifterParticle, - lat=[drifter.spacetime.location.lat for drifter in drifters], - lon=[drifter.spacetime.location.lon for drifter in drifters], - depth=[drifter.depth for drifter in drifters], - time=[drifter.spacetime.time for drifter in drifters], + lat=[drifter.spacetime.location.lat for drifter in self.measurements], + lon=[drifter.spacetime.location.lon for drifter in self.measurements], + depth=[drifter.depth for drifter in self.measurements], + time=[drifter.spacetime.time for drifter in self.measurements], has_lifetime=[ - 1 if drifter.lifetime is not None else 0 for drifter in drifters + 1 if drifter.lifetime is not None else 0 + for drifter in self.measurements ], lifetime=[ 0 if drifter.lifetime is None else drifter.lifetime.total_seconds() - for drifter in drifters + for drifter in self.measurements ], ) # define output file for the simulation out_file = drifter_particleset.ParticleFile( - name=out_path, outputdt=outputdt, chunks=[len(drifter_particleset), 100] + name=self.out_path, + outputdt=OUTPUT_DT, + chunks=[len(drifter_particleset), 100], ) # get earliest between fieldset end time and provide end time fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - if endtime is None: + if ENDTIME is None: actual_endtime = fieldset_endtime - elif endtime > fieldset_endtime: + elif ENDTIME > fieldset_endtime: print("WARN: Requested end time later than fieldset end time.") actual_endtime = fieldset_endtime else: - actual_endtime = np.timedelta64(endtime) + actual_endtime = np.timedelta64(ENDTIME) # execute simulation drifter_particleset.execute( [AdvectionRK4, _sample_temperature, _check_lifetime], endtime=actual_endtime, - dt=dt, + dt=DT, output_file=out_file, verbose_progress=True, ) # if there are more particles left than the number of drifters with an indefinite endtime, warn the user if len(drifter_particleset.particledata) > len( - [d for d in drifters if d.lifetime is None] + [d for d in self.measurements if d.lifetime is None] ): print( "WARN: Some drifters had a life time beyond the end time of the fieldset or the requested end time." diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 371b4485..9c0fc401 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from pathlib import Path from typing import ClassVar import numpy as np @@ -7,7 +6,6 @@ from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType -from virtualship.models.spacetime import Spacetime from virtualship.utils import register_input_dataset, register_instrument @@ -80,32 +78,28 @@ def get_datasets_dict(self) -> dict: class Underwater_STInstrument(Instrument): """Underwater_ST instrument class.""" - def __init__( - self, - input_dataset: InputDataset, - ): + def __init__(self, name, expedition, directory): """Initialize Underwater_STInstrument.""" filenames = { - "S": input_dataset.data_dir.joinpath(f"{input_dataset.name}_s.nc"), - "T": input_dataset.data_dir.joinpath(f"{input_dataset.name}_t.nc"), + "S": directory.joinpath(f"{name}_s.nc"), + "T": directory.joinpath(f"{name}_t.nc"), } variables = {"S": "so", "T": "thetao"} super().__init__( - input_dataset, + Underwater_ST.name, + expedition, + directory, filenames, variables, add_bathymetry=False, allow_time_extrapolation=True, ) - def simulate( - self, - out_path: str | Path, - depth: float, - sample_points: list[Spacetime], - ) -> None: + def simulate(self) -> None: """Simulate underway salinity and temperature measurements.""" - sample_points.sort(key=lambda p: p.time) + DEPTH = -2.0 + + self.measurements.sort(key=lambda p: p.time) fieldset = self.load_input_data() @@ -114,13 +108,13 @@ def simulate( pclass=_ShipSTParticle, lon=0.0, lat=0.0, - depth=depth, + depth=DEPTH, time=0, ) - out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) + out_file = particleset.ParticleFile(name=self.out_path, outputdt=np.inf) - for point in sample_points: + for point in self.measurements: particleset.lon_nextloop[:] = point.location.lon particleset.lat_nextloop[:] = point.location.lat particleset.time_nextloop[:] = fieldset.time_origin.reltime( diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index c2fec98d..7c916d98 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -1,6 +1,5 @@ from dataclasses import dataclass from datetime import timedelta -from pathlib import Path from typing import ClassVar import numpy as np @@ -87,17 +86,17 @@ def get_datasets_dict(self) -> dict: "UVdata": { "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", "variables": ["uo", "vo"], - "output_filename": "ship_uv.nc", + "output_filename": f"{self.name}_uv.nc", }, "Sdata": { "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", "variables": ["so"], - "output_filename": "ship_s.nc", + "output_filename": f"{self.name}_s.nc", }, "Tdata": { "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", "variables": ["thetao"], - "output_filename": "ship_t.nc", + "output_filename": f"{self.name}_t.nc", }, } @@ -106,35 +105,30 @@ def get_datasets_dict(self) -> dict: class XBTInstrument(Instrument): """XBT instrument class.""" - def __init__( - self, - input_dataset: InputDataset, - ): + def __init__(self, name, expedition, directory): """Initialize XBTInstrument.""" filenames = { - "UV": input_dataset.data_dir.joinpath("ship_uv.nc"), - "S": input_dataset.data_dir.joinpath("ship_s.nc"), - "T": input_dataset.data_dir.joinpath("ship_t.nc"), + "UV": directory.joinpath(f"{name}_uv.nc"), + "S": directory.joinpath(f"{name}_s.nc"), + "T": directory.joinpath(f"{name}_t.nc"), } variables = {"UV": ["uo", "vo"], "S": "so", "T": "thetao"} super().__init__( - input_dataset, + XBT.name, + expedition, + directory, filenames, variables, add_bathymetry=True, allow_time_extrapolation=True, ) - def simulate( - self, - xbts: list[XBT], - out_path: str | Path, - outputdt: timedelta, - ) -> None: + def simulate(self) -> None: """Simulate XBT measurements.""" DT = 10.0 # dt of XBT simulation integrator + OUTPUT_DT = timedelta(seconds=1) - if len(xbts) == 0: + if len(self.measurements) == 0: print( "No XBTs provided. Parcels currently crashes when providing an empty particle set, so no XBT simulation will be done and no files will be created." ) @@ -148,7 +142,10 @@ def simulate( # deploy time for all xbts should be later than fieldset start time if not all( - [np.datetime64(xbt.spacetime.time) >= fieldset_starttime for xbt in xbts] + [ + np.datetime64(xbt.spacetime.time) >= fieldset_starttime + for xbt in self.measurements + ] ): raise ValueError("XBT deployed before fieldset starts.") @@ -163,11 +160,11 @@ def simulate( time=0, ), ) - for xbt in xbts + for xbt in self.measurements ] # initial fall speeds - initial_fall_speeds = [xbt.fall_speed for xbt in xbts] + initial_fall_speeds = [xbt.fall_speed for xbt in self.measurements] # XBT depth can not be too shallow, because kernel would break. for max_depth, fall_speed in zip(max_depths, initial_fall_speeds, strict=False): @@ -180,16 +177,16 @@ def simulate( xbt_particleset = ParticleSet( fieldset=fieldset, pclass=_XBTParticle, - lon=[xbt.spacetime.location.lon for xbt in xbts], - lat=[xbt.spacetime.location.lat for xbt in xbts], - depth=[xbt.min_depth for xbt in xbts], - time=[xbt.spacetime.time for xbt in xbts], + lon=[xbt.spacetime.location.lon for xbt in self.measurements], + lat=[xbt.spacetime.location.lat for xbt in self.measurements], + depth=[xbt.min_depth for xbt in self.measurements], + time=[xbt.spacetime.time for xbt in self.measurements], max_depth=max_depths, - min_depth=[xbt.min_depth for xbt in xbts], - fall_speed=[xbt.fall_speed for xbt in xbts], + min_depth=[xbt.min_depth for xbt in self.measurements], + fall_speed=[xbt.fall_speed for xbt in self.measurements], ) - out_file = xbt_particleset.ParticleFile(name=out_path, outputdt=outputdt) + out_file = xbt_particleset.ParticleFile(name=self.out_path, outputdt=OUTPUT_DT) xbt_particleset.execute( [_sample_temperature, _xbt_cast], From 3efa29bda70de2e3a22a287321142033c2fb5a7f Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:24:53 +0100 Subject: [PATCH 30/97] draft up check land using bathymetry --- src/virtualship/expedition/do_expedition.py | 4 +- src/virtualship/instruments/base.py | 6 -- src/virtualship/models/expedition.py | 70 +++++++++------------ 3 files changed, 31 insertions(+), 49 deletions(-) diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index c4bc6783..7be947b2 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -48,10 +48,8 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> # verify schedule is valid # TODO: needs updating when .verify() updated to not need input_data - - loaded_input_data = [] # TODO: TEMPORARY! expedition.schedule.verify( - expedition.ship_config.ship_speed_knots, loaded_input_data + expedition.ship_config.ship_speed_knots, input_dir=input_data ) # simulate the schedule diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 13a2fb05..001dc8c5 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -99,14 +99,8 @@ def __init__( def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" - # TODO: this should mean can delete input_data.py! - - # TODO: hopefully simulate_measurements can also be removed! And maybe the list of e.g. ctds ('measurements') to run can be added to higher level like do_expedition.py...? I think as they already do... - # TODO: can simulate_schedule.py be refactored to be contained in base.py and repsective instrument files too...? - # TODO: what do I need to do about automatic registration of Instrument classes...? - # TODO: tests will need updating...! # TODO: think about combining InputDataset and Instrument classes together! Or maybe not if they are better kept separate... diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 9add2c1b..8888f657 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -2,12 +2,15 @@ import itertools from datetime import datetime, timedelta +from pathlib import Path from typing import TYPE_CHECKING +import numpy as np import pydantic import pyproj import yaml +from parcels import Field from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.instruments.types import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta @@ -87,9 +90,9 @@ class Schedule(pydantic.BaseModel): def verify( self, ship_speed: float, + input_dir: str | Path | None, *, check_space_time_region: bool = False, - ignore_missing_fieldsets: bool = False, ) -> None: """ Verify the feasibility and correctness of the schedule's waypoints. @@ -102,9 +105,9 @@ def verify( 5. The ship can arrive on time at each waypoint given its speed. :param ship_speed: The ship's speed in knots. - :param input_data: An InputData object containing fieldsets used to check if waypoints are on water. + :param input_dir: The input directory containing necessary files. :param check_space_time_region: whether to check for missing space_time_region. - :param ignore_missing_fieldsets: whether to ignore warning for missing field sets. + # :param ignore_missing_fieldsets: whether to ignore warning for missing field sets. :raises PlanningError: If any of the verification checks fail, indicating infeasible or incorrect waypoints. :raises NotImplementedError: If an instrument in the schedule is not implemented. :return: None. The method doesn't return a value but raises exceptions if verification fails. @@ -134,46 +137,33 @@ def verify( f"Waypoint(s) {', '.join(f'#{i + 1}' for i in invalid_i)}: each waypoint should be timed after all previous waypoints", ) - # check if all waypoints are in water - # this is done by picking an arbitrary provided fieldset and checking if UV is not zero - - # TODO: this may need to be done with generic bathymetry data, now that removed InputData! - # get all available fieldsets - available_fieldsets = [] - # if input_data is not None: - # fieldsets = [ - # input_data.adcp_fieldset, - # input_data.argo_float_fieldset, - # input_data.ctd_fieldset, - # input_data.drifter_fieldset, - # input_data.ship_underwater_st_fieldset, - # ] - # for fs in fieldsets: - # if fs is not None: - # available_fieldsets.append(fs) - - # check if there are any fieldsets, else it's an error - if len(available_fieldsets) == 0: - if not ignore_missing_fieldsets: - print( - "Cannot verify because no fieldsets have been loaded. This is probably " - "because you are not using any instruments in your schedule. This is not a problem, " - "but carefully check your waypoint locations manually." + # check if all waypoints are in water using bathymetry data + # TODO: tests should be updated to check this! + land_waypoints = [] + if input_dir is not None: + bathymetry_path = input_dir.joinpath("bathymetry.nc") + try: + bathymetry_field = Field.from_netcdf( + bathymetry_path, + variables=("bathymetry", "deptho"), + dimensions={"lon": "longitude", "lat": "latitude"}, ) - - else: - # pick any - fieldset = available_fieldsets[0] - # get waypoints with 0 UV - land_waypoints = [ - (wp_i, wp) - for wp_i, wp in enumerate(self.waypoints) - if _is_on_land_zero_uv(fieldset, wp) - ] - # raise an error if there are any + except Exception as e: + raise FileNotFoundError( + "Bathymetry file not found in input data. Cannot verify waypoints are in water." + ) from e + for wp_i, wp in enumerate(self.waypoints): + bathy = bathymetry_field.eval( + 0, # time + 0, # depth (surface) + wp.location.lat, + wp.location.lon, + ) + if np.isnan(bathy) or bathy >= 0: + land_waypoints.append((wp_i, wp)) if len(land_waypoints) > 0: raise ScheduleError( - f"The following waypoints are on land: {['#' + str(wp_i) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" + f"The following waypoints are on land: {['#' + str(wp_i + 1) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" ) # check that ship will arrive on time at each waypoint (in case no unexpected event happen) From aa2d3095be32b63695e146e7dedfcdb2c3985eef Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:55:44 +0100 Subject: [PATCH 31/97] small bug fixes --- src/virtualship/cli/_fetch.py | 3 +-- src/virtualship/cli/_plan.py | 7 ++++--- src/virtualship/expedition/__init__.py | 2 -- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 67695695..e472eb9a 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -52,9 +52,8 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None expedition.schedule.verify( expedition.ship_config.ship_speed_knots, - input_data=None, + input_dir=None, check_space_time_region=True, - ignore_missing_fieldsets=True, ) space_time_region_hash = get_space_time_region_hash( diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index a071c38e..41709a5c 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -951,7 +951,9 @@ def copy_from_previous(self) -> None: if prev and curr: curr.value = prev.value - for instrument in InstrumentType: + for instrument in [ + inst for inst in InstrumentType if not inst.is_underway + ]: prev_switch = schedule_editor.query_one( f"#wp{self.index - 1}_{instrument.value}" ) @@ -1044,9 +1046,8 @@ def save_pressed(self) -> None: # verify schedule expedition_editor.expedition.schedule.verify( ship_speed_value, - input_data=None, + input_dir=None, check_space_time_region=True, - ignore_missing_fieldsets=True, ) expedition_saved = expedition_editor.save_changes() diff --git a/src/virtualship/expedition/__init__.py b/src/virtualship/expedition/__init__.py index 43d24844..dfa61028 100644 --- a/src/virtualship/expedition/__init__.py +++ b/src/virtualship/expedition/__init__.py @@ -1,9 +1,7 @@ """Everything for simulating an expedition.""" from .do_expedition import do_expedition -from .input_data import InputData __all__ = [ - "InputData", "do_expedition", ] From 588cab4fc6fde2eee6c3e73c80b53f778786b54e Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 29 Oct 2025 11:23:28 +0100 Subject: [PATCH 32/97] patch copernicus product id search logic to new instrument classes, plus more debugging; verbose INFO is outstanding --- src/virtualship/cli/_fetch.py | 141 +------------ src/virtualship/cli/_plan.py | 3 +- src/virtualship/expedition/do_expedition.py | 29 ++- .../expedition/simulate_schedule.py | 22 ++- src/virtualship/instruments/adcp.py | 23 ++- src/virtualship/instruments/argo_float.py | 43 ++-- src/virtualship/instruments/base.py | 185 ++++++++++++++++-- src/virtualship/instruments/ctd.py | 23 +-- src/virtualship/instruments/ctd_bgc.py | 47 ++--- src/virtualship/instruments/drifter.py | 32 +-- .../instruments/ship_underwater_st.py | 19 +- src/virtualship/instruments/xbt.py | 27 +-- src/virtualship/models/expedition.py | 32 +-- 13 files changed, 339 insertions(+), 287 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index e472eb9a..a41687d0 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -7,11 +7,10 @@ from typing import TYPE_CHECKING import copernicusmarine -import numpy as np from copernicusmarine.core_functions.credentials_utils import InvalidUsernameOrPassword from pydantic import BaseModel -from virtualship.errors import CopernicusCatalogueError, IncompleteDownloadError +from virtualship.errors import IncompleteDownloadError from virtualship.utils import ( _dump_yaml, _generic_load_yaml, @@ -52,8 +51,9 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None expedition.schedule.verify( expedition.ship_config.ship_speed_knots, - input_dir=None, + data_dir=None, check_space_time_region=True, + ignore_missing_bathymetry=True, ) space_time_region_hash = get_space_time_region_hash( @@ -107,7 +107,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None coordinates_selection_method="outside", ) - # Only keep instruments present in the expedition + # download, only instruments present in the expedition for itype in instruments_in_expedition: input_dataset_class = get_input_dataset_class(itype) if input_dataset_class is None: @@ -225,136 +225,3 @@ def complete_download(download_path: Path) -> None: metadata = DownloadMetadata(download_complete=True, download_date=datetime.now()) metadata.to_yaml(download_metadata) return - - -def select_product_id( - physical: bool, - schedule_start: datetime, - schedule_end: datetime, - username: str, - password: str, - variable: str | None = None, # only needed for BGC datasets -) -> str: - """ - Determine which copernicus product id should be selected (reanalysis, reanalysis-interim, analysis & forecast), for prescribed schedule and physical vs. BGC. - - BGC is more complicated than physical products. Often (re)analysis period and variable dependent, hence more custom logic here. - """ - product_ids = { - "phys": { - "reanalysis": "cmems_mod_glo_phy_my_0.083deg_P1D-m", - "reanalysis_interim": "cmems_mod_glo_phy_myint_0.083deg_P1D-m", - "analysis": "cmems_mod_glo_phy_anfc_0.083deg_P1D-m", - }, - "bgc": { - "reanalysis": "cmems_mod_glo_bgc_my_0.25deg_P1D-m", - "reanalysis_interim": "cmems_mod_glo_bgc_myint_0.25deg_P1D-m", - "analysis": None, # will be set per variable - }, - } - - bgc_analysis_ids = { - "o2": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", - "chl": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", - "no3": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", - "po4": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", - "ph": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", - "phyc": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", - "nppv": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", - } - - # pH and phytoplankton variables are available as *monthly* products only in renalysis(_interim) period - monthly_bgc_reanalysis_ids = { - "ph": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", - "phyc": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", - } - monthly_bgc_reanalysis_interim_ids = { - "ph": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", - "phyc": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", - } - - key = "phys" if physical else "bgc" - selected_id = None - - for period, pid in product_ids[key].items(): - # for BGC analysis, set pid per variable - if key == "bgc" and period == "analysis": - if variable is None or variable not in bgc_analysis_ids: - continue - pid = bgc_analysis_ids[variable] - # for BGC reanalysis, check if requires monthly product - if ( - key == "bgc" - and period == "reanalysis" - and variable in monthly_bgc_reanalysis_ids - ): - monthly_pid = monthly_bgc_reanalysis_ids[variable] - ds_monthly = copernicusmarine.open_dataset( - monthly_pid, - username=username, - password=password, - ) - time_end_monthly = ds_monthly["time"][-1].values - if np.datetime64(schedule_end) <= time_end_monthly: - pid = monthly_pid - # for BGC reanalysis_interim, check if requires monthly product - if ( - key == "bgc" - and period == "reanalysis_interim" - and variable in monthly_bgc_reanalysis_interim_ids - ): - monthly_pid = monthly_bgc_reanalysis_interim_ids[variable] - ds_monthly = copernicusmarine.open_dataset( - monthly_pid, username=username, password=password - ) - time_end_monthly = ds_monthly["time"][-1].values - if np.datetime64(schedule_end) <= time_end_monthly: - pid = monthly_pid - if pid is None: - continue - ds = copernicusmarine.open_dataset(pid, username=username, password=password) - time_end = ds["time"][-1].values - if np.datetime64(schedule_end) <= time_end: - selected_id = pid - break - - if selected_id is None: - raise CopernicusCatalogueError( - "No suitable product found in the Copernicus Marine Catalogue for the scheduled time and variable." - ) - - # handle the rare situation where start time and end time span different products, which is possible for reanalysis and reanalysis_interim - # in this case, return the analysis product which spans far back enough - if start_end_in_product_timerange( - selected_id, schedule_start, schedule_end, username, password - ): - return selected_id - - else: - return ( - product_ids["phys"]["analysis"] if physical else bgc_analysis_ids[variable] - ) - - -def start_end_in_product_timerange( - selected_id: str, - schedule_start: datetime, - schedule_end: datetime, - username: str, - password: str, -) -> bool: - """Check schedule_start and schedule_end are both within a selected Copernicus product's time range.""" - ds_selected = copernicusmarine.open_dataset( - selected_id, username=username, password=password - ) - time_values = ds_selected["time"].values - time_min, time_max = np.min(time_values), np.max(time_values) - - if ( - np.datetime64(schedule_start) >= time_min - and np.datetime64(schedule_end) <= time_max - ): - return True - - else: - return False diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 41709a5c..c27cb741 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -1046,8 +1046,9 @@ def save_pressed(self) -> None: # verify schedule expedition_editor.expedition.schedule.verify( ship_speed_value, - input_dir=None, + data_dir=None, check_space_time_region=True, + ignore_missing_bathymetry=True, ) expedition_saved = expedition_editor.save_changes() diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index 7be947b2..ec61b1b0 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -6,23 +6,31 @@ import pyproj +from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash from virtualship.models import Schedule -from virtualship.utils import CHECKPOINT, _get_expedition, get_instrument_class +from virtualship.utils import ( + CHECKPOINT, + _get_expedition, + get_instrument_class, +) from .checkpoint import Checkpoint from .expedition_cost import expedition_cost -from .simulate_schedule import ScheduleProblem, simulate_schedule +from .simulate_schedule import ( + MeasurementsToSimulate, + ScheduleProblem, + simulate_schedule, +) # projection used to sail between waypoints projection = pyproj.Geod(ellps="WGS84") -def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> None: +def do_expedition(expedition_dir: str | Path) -> None: """ Perform an expedition, providing terminal feedback and file output. :param expedition_dir: The base directory for the expedition. - :param input_data: Input data folder (override used for testing). """ print("\n╔═════════════════════════════════════════════════╗") print("║ VIRTUALSHIP EXPEDITION STATUS ║") @@ -47,11 +55,13 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> print("\n---- WAYPOINT VERIFICATION ----") # verify schedule is valid - # TODO: needs updating when .verify() updated to not need input_data - expedition.schedule.verify( - expedition.ship_config.ship_speed_knots, input_dir=input_data + data_dir = get_existing_download( + expedition_dir, + get_space_time_region_hash(expedition.schedule.space_time_region), ) + expedition.schedule.verify(expedition.ship_config.ship_speed_knots, data_dir) + # simulate the schedule schedule_results = simulate_schedule( projection=projection, @@ -96,8 +106,9 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> if instrument_class is None: raise RuntimeError(f"No instrument class found for type {itype}.") - # get measurements to simulate for this instrument - measurements = schedule_results.measurements_to_simulate.get(itype.name.lower()) + # get measurements to simulate + attr = MeasurementsToSimulate.get_attr_for_instrumenttype(itype) + measurements = getattr(schedule_results.measurements_to_simulate, attr) # initialise instrument instrument = instrument_class(expedition=expedition, directory=expedition_dir) diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index f8d142ea..3b8fa78a 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta +from typing import ClassVar import pyproj @@ -39,7 +40,26 @@ class ScheduleProblem: @dataclass class MeasurementsToSimulate: - """The measurements to simulate, as concluded from schedule simulation.""" + """ + The measurements to simulate, as concluded from schedule simulation. + + Provides a mapping from InstrumentType to the correct attribute name for robust access. + """ + + _instrumenttype_to_attr: ClassVar[dict] = { + InstrumentType.ADCP: "adcps", + InstrumentType.UNDERWATER_ST: "ship_underwater_sts", + InstrumentType.ARGO_FLOAT: "argo_floats", + InstrumentType.DRIFTER: "drifters", + InstrumentType.CTD: "ctds", + InstrumentType.CTD_BGC: "ctd_bgcs", + InstrumentType.XBT: "xbts", + } + + @classmethod + def get_attr_for_instrumenttype(cls, instrument_type): + """Return the attribute name for a given InstrumentType.""" + return cls._instrumenttype_to_attr[instrument_type] adcps: list[Spacetime] = field(default_factory=list, init=False) ship_underwater_sts: list[Spacetime] = field(default_factory=list, init=False) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 917da154..db5be8ef 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -63,7 +63,7 @@ def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["uo", "vo"], "output_filename": f"{self.name}_uv.nc", }, @@ -74,12 +74,13 @@ def get_datasets_dict(self) -> dict: class ADCPInstrument(Instrument): """ADCP instrument class.""" - def __init__(self, name, expedition, directory): + def __init__(self, expedition, directory): """Initialize ADCPInstrument.""" filenames = { - "UV": directory.joinpath(f"{name}_uv.nc"), + "U": f"{ADCP.name}_uv.nc", + "V": f"{ADCP.name}_uv.nc", } - variables = {"UV": ["uo", "vo"]} + variables = {"U": "uo", "V": "vo"} super().__init__( ADCP.name, expedition, @@ -90,13 +91,13 @@ def __init__(self, name, expedition, directory): allow_time_extrapolation=True, ) - def simulate(self) -> None: + def simulate(self, measurements, out_path) -> None: """Simulate ADCP measurements.""" MAX_DEPTH = self.expedition.instruments_config.adcp_config.max_depth_meter MIN_DEPTH = -5.0 - NUM_BINS = self.instruments_config.adcp_config.num_bins + NUM_BINS = self.expedition.instruments_config.adcp_config.num_bins - self.measurements.sort(key=lambda p: p.time) + measurements.sort(key=lambda p: p.time) fieldset = self.load_input_data() @@ -105,15 +106,17 @@ def simulate(self) -> None: particleset = ParticleSet.from_list( fieldset=fieldset, pclass=_ADCPParticle, - lon=np.full(num_particles, 0.0), + lon=np.full( + num_particles, 0.0 + ), # initial lat/lon are irrelevant and will be overruled later.s lat=np.full(num_particles, 0.0), depth=bins, time=0, ) - out_file = particleset.ParticleFile(name=self.out_path, outputdt=np.inf) + out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) - for point in self.measurements: + for point in measurements: particleset.lon_nextloop[:] = point.location.lon particleset.lat_nextloop[:] = point.location.lat particleset.time_nextloop[:] = fieldset.time_origin.reltime( diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 239b47de..93aa9093 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -144,17 +144,17 @@ def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["uo", "vo"], "output_filename": f"{self.name}_uv.nc", }, "Sdata": { - "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["so"], "output_filename": f"{self.name}_s.nc", }, "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["thetao"], "output_filename": f"{self.name}_t.nc", }, @@ -165,14 +165,15 @@ def get_datasets_dict(self) -> dict: class ArgoFloatInstrument(Instrument): """ArgoFloat instrument class.""" - def __init__(self, name, expedition, directory): + def __init__(self, expedition, directory): """Initialize ArgoFloatInstrument.""" filenames = { - "UV": directory.joinpath(f"{name}_uv.nc"), - "S": directory.joinpath(f"{name}_s.nc"), - "T": directory.joinpath(f"{name}_t.nc"), + "U": f"{ArgoFloat.name}_uv.nc", + "V": f"{ArgoFloat.name}_uv.nc", + "S": f"{ArgoFloat.name}_s.nc", + "T": f"{ArgoFloat.name}_t.nc", } - variables = {"UV": ["uo", "vo"], "S": "so", "T": "thetao"} + variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} super().__init__( ArgoFloat.name, expedition, @@ -183,13 +184,13 @@ def __init__(self, name, expedition, directory): allow_time_extrapolation=False, ) - def simulate(self) -> None: + def simulate(self, measurements, out_path) -> None: """Simulate Argo float measurements.""" DT = 10.0 # dt of Argo float simulation integrator OUTPUT_DT = timedelta(minutes=5) ENDTIME = None - if len(self.measurements) == 0: + if len(measurements) == 0: print( "No Argo floats provided. Parcels currently crashes when providing an empty particle set, so no argo floats simulation will be done and no files will be created." ) @@ -202,21 +203,21 @@ def simulate(self) -> None: argo_float_particleset = ParticleSet( fieldset=fieldset, pclass=_ArgoParticle, - lat=[argo.spacetime.location.lat for argo in self.measurements], - lon=[argo.spacetime.location.lon for argo in self.measurements], - depth=[argo.min_depth for argo in self.measurements], - time=[argo.spacetime.time for argo in self.measurements], - min_depth=[argo.min_depth for argo in self.measurements], - max_depth=[argo.max_depth for argo in self.measurements], - drift_depth=[argo.drift_depth for argo in self.measurements], - vertical_speed=[argo.vertical_speed for argo in self.measurements], - cycle_days=[argo.cycle_days for argo in self.measurements], - drift_days=[argo.drift_days for argo in self.measurements], + lat=[argo.spacetime.location.lat for argo in measurements], + lon=[argo.spacetime.location.lon for argo in measurements], + depth=[argo.min_depth for argo in measurements], + time=[argo.spacetime.time for argo in measurements], + min_depth=[argo.min_depth for argo in measurements], + max_depth=[argo.max_depth for argo in measurements], + drift_depth=[argo.drift_depth for argo in measurements], + vertical_speed=[argo.vertical_speed for argo in measurements], + cycle_days=[argo.cycle_days for argo in measurements], + drift_days=[argo.drift_days for argo in measurements], ) # define output file for the simulation out_file = argo_float_particleset.ParticleFile( - name=self.out_path, + name=out_path, outputdt=OUTPUT_DT, chunks=[len(argo_float_particleset), 100], ) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 001dc8c5..6ae38cc0 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -3,12 +3,47 @@ from pathlib import Path import copernicusmarine -import yaspin +import numpy as np +from yaspin import yaspin from parcels import Field, FieldSet +from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash +from virtualship.errors import CopernicusCatalogueError from virtualship.models import Expedition, SpaceTimeRegion from virtualship.utils import ship_spinner +PRODUCT_IDS = { + "phys": { + "reanalysis": "cmems_mod_glo_phy_my_0.083deg_P1D-m", + "reanalysis_interim": "cmems_mod_glo_phy_myint_0.083deg_P1D-m", + "analysis": "cmems_mod_glo_phy_anfc_0.083deg_P1D-m", + }, + "bgc": { + "reanalysis": "cmems_mod_glo_bgc_my_0.25deg_P1D-m", + "reanalysis_interim": "cmems_mod_glo_bgc_myint_0.25deg_P1D-m", + "analysis": None, # will be set per variable + }, +} + +BGC_ANALYSIS_IDS = { + "o2": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "chl": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "no3": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "po4": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "ph": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", + "phyc": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "nppv": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", +} + +MONTHLY_BGC_REANALYSIS_IDS = { + "ph": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", + "phyc": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", +} +MONTHLY_BGC_REANALYSIS_INTERIM_IDS = { + "ph": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", + "phyc": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", +} + class InputDataset(abc.ABC): """Base class for instrument input datasets.""" @@ -39,7 +74,7 @@ def get_datasets_dict(self) -> dict: """Get parameters for instrument's variable(s) specific data download.""" def download_data(self) -> None: - """Download data for the instrument using copernicusmarine.""" + """Download data for the instrument using copernicusmarine, with correct product ID selection.""" parameter_args = dict( minimum_longitude=self.space_time_region.spatial_range.minimum_longitude - self.latlon_buffer, @@ -62,10 +97,117 @@ def download_data(self) -> None: ) datasets_args = self.get_datasets_dict() + for dataset in datasets_args.values(): - download_args = {**parameter_args, **dataset} + physical = dataset.get("physical") + if physical: + variable = None + else: + variable = dataset.get("variables")[0] # BGC variables, special case + + dataset_id = self._select_product_id( + physical=physical, + schedule_start=self.space_time_region.time_range.start_time, + schedule_end=self.space_time_region.time_range.end_time, + username=self.credentials["username"], + password=self.credentials["password"], + variable=variable, + ) + download_args = { + **parameter_args, + **{k: v for k, v in dataset.items() if k != "physical"}, + "dataset_id": dataset_id, + } copernicusmarine.subset(**download_args) + def _select_product_id( + self, + physical: bool, + schedule_start, + schedule_end, + username: str, + password: str, + variable: str | None = None, + ) -> str: + """Determine which copernicus product id should be selected (reanalysis, reanalysis-interim, analysis & forecast), for prescribed schedule and physical vs. BGC.""" + key = "phys" if physical else "bgc" + selected_id = None + + for period, pid in PRODUCT_IDS[key].items(): + # for BGC analysis, set pid per variable + if key == "bgc" and period == "analysis": + if variable is None or variable not in BGC_ANALYSIS_IDS: + continue + pid = BGC_ANALYSIS_IDS[variable] + # for BGC reanalysis, check if requires monthly product + if ( + key == "bgc" + and period == "reanalysis" + and variable in MONTHLY_BGC_REANALYSIS_IDS + ): + monthly_pid = MONTHLY_BGC_REANALYSIS_IDS[variable] + ds_monthly = copernicusmarine.open_dataset( + monthly_pid, + username=username, + password=password, + ) + time_end_monthly = ds_monthly["time"][-1].values + if np.datetime64(schedule_end) <= time_end_monthly: + pid = monthly_pid + # for BGC reanalysis_interim, check if requires monthly product + if ( + key == "bgc" + and period == "reanalysis_interim" + and variable in MONTHLY_BGC_REANALYSIS_INTERIM_IDS + ): + monthly_pid = MONTHLY_BGC_REANALYSIS_INTERIM_IDS[variable] + ds_monthly = copernicusmarine.open_dataset( + monthly_pid, username=username, password=password + ) + time_end_monthly = ds_monthly["time"][-1].values + if np.datetime64(schedule_end) <= time_end_monthly: + pid = monthly_pid + if pid is None: + continue + ds = copernicusmarine.open_dataset( + pid, username=username, password=password + ) + time_end = ds["time"][-1].values + if np.datetime64(schedule_end) <= time_end: + selected_id = pid + break + + if selected_id is None: + raise CopernicusCatalogueError( + "No suitable product found in the Copernicus Marine Catalogue for the scheduled time and variable." + ) + + def start_end_in_product_timerange( + selected_id, schedule_start, schedule_end, username, password + ): + ds_selected = copernicusmarine.open_dataset( + selected_id, username=username, password=password + ) + time_values = ds_selected["time"].values + import numpy as np + + time_min, time_max = np.min(time_values), np.max(time_values) + return ( + np.datetime64(schedule_start) >= time_min + and np.datetime64(schedule_end) <= time_max + ) + + if start_end_in_product_timerange( + selected_id, schedule_start, schedule_end, username, password + ): + return selected_id + else: + return ( + PRODUCT_IDS["phys"]["analysis"] + if physical + else BGC_ANALYSIS_IDS[variable] + ) + class Instrument(abc.ABC): """Base class for instruments and their simulation.""" @@ -93,21 +235,23 @@ def __init__( "time": "time", "depth": "depth", } # same dimensions for all instruments - self.bathymetry_file = self.directory.joinpath(bathymetry_file) + self.bathymetry_file = bathymetry_file self.add_bathymetry = add_bathymetry self.allow_time_extrapolation = allow_time_extrapolation def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" # TODO: can simulate_schedule.py be refactored to be contained in base.py and repsective instrument files too...? - - # TODO: tests will need updating...! - - # TODO: think about combining InputDataset and Instrument classes together! Or maybe not if they are better kept separate... + # TODO: tests need updating...! try: + data_dir = self._get_data_dir(self.directory) + joined_filepaths = { + key: data_dir.joinpath(filename) + for key, filename in self.filenames.items() + } fieldset = FieldSet.from_netcdf( - self.filenames, + joined_filepaths, self.variables, self.dimensions, allow_time_extrapolation=self.allow_time_extrapolation, @@ -118,7 +262,7 @@ def load_input_data(self) -> FieldSet: ) from e # interpolation methods - for var in self.variables: + for var in (v for v in self.variables if v not in ("U", "V")): getattr(fieldset, var).interp_method = "linear_invdist_land_tracer" # depth negative for g in fieldset.gridset.grids: @@ -126,17 +270,18 @@ def load_input_data(self) -> FieldSet: # bathymetry data if self.add_bathymetry: bathymetry_field = Field.from_netcdf( - self.bathymetry_file, - self.bathymetry_variables, - self.bathymetry_dimensions, + data_dir.joinpath(self.bathymetry_file), + variable=("bathymetry", "deptho"), + dimensions={"lon": "longitude", "lat": "latitude"}, ) bathymetry_field.data = -bathymetry_field.data fieldset.add_field(bathymetry_field) fieldset.computeTimeChunk(0, 1) # read in data already + return fieldset @abc.abstractmethod - def simulate(self, measurements: list, out_path: str | Path): + def simulate(self, data_dir: Path, measurements: list, out_path: str | Path): """Simulate instrument measurements.""" def run(self, measurements: list, out_path: str | Path) -> None: @@ -150,3 +295,15 @@ def run(self, measurements: list, out_path: str | Path) -> None: ) as spinner: self.simulate(measurements, out_path) spinner.ok("✅") + + def _get_data_dir(self, expedition_dir: Path) -> Path: + space_time_region_hash = get_space_time_region_hash( + self.expedition.schedule.space_time_region + ) + data_dir = get_existing_download(expedition_dir, space_time_region_hash) + + assert data_dir is not None, ( + "Input data hasn't been found. Have you run the `virtualship fetch` command?" + ) + + return data_dir diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 82cace94..c1981e06 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -8,7 +8,7 @@ from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models import Spacetime -from virtualship.utils import register_input_dataset +from virtualship.utils import register_input_dataset, register_instrument @dataclass @@ -81,26 +81,27 @@ def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { "Sdata": { - "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["so"], "output_filename": f"{self.name}_s.nc", }, "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["thetao"], "output_filename": f"{self.name}_t.nc", }, } +@register_instrument(InstrumentType.CTD) class CTDInstrument(Instrument): """CTD instrument class.""" - def __init__(self, name, expedition, directory): + def __init__(self, expedition, directory): """Initialize CTDInstrument.""" filenames = { - "S": directory.data_dir.joinpath(f"{name}_s.nc"), - "T": directory.data_dir.joinpath(f"{name}_t.nc"), + "S": f"{CTD.name}_s.nc", + "T": f"{CTD.name}_t.nc", } variables = {"S": "so", "T": "thetao"} @@ -114,13 +115,13 @@ def __init__(self, name, expedition, directory): allow_time_extrapolation=True, ) - def simulate(self) -> None: + def simulate(self, measurements, out_path) -> None: """Simulate CTD measurements.""" WINCH_SPEED = 1.0 # sink and rise speed in m/s DT = 10.0 # dt of CTD simulation integrator OUTPUT_DT = timedelta(seconds=10) # output dt for CTD simulation - if len(self.measurements) == 0: + if len(measurements) == 0: print( "No CTDs provided. Parcels currently crashes when providing an empty particle set, so no CTD simulation will be done and no files will be created." ) @@ -136,7 +137,7 @@ def simulate(self) -> None: if not all( [ np.datetime64(ctd.spacetime.time) >= fieldset_starttime - for ctd in self.measurements + for ctd in measurements ] ): raise ValueError("CTD deployed before fieldset starts.") @@ -152,7 +153,7 @@ def simulate(self) -> None: time=0, ), ) - for ctd in self.measurements + for ctd in measurements ] # CTD depth can not be too shallow, because kernel would break. @@ -176,7 +177,7 @@ def simulate(self) -> None: ) # define output file for the simulation - out_file = ctd_particleset.ParticleFile(name=self.out_path, outputdt=OUTPUT_DT) + out_file = ctd_particleset.ParticleFile(name=out_path, outputdt=OUTPUT_DT) # execute simulation ctd_particleset.execute( diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index b017e0f4..ed9ab9d9 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -106,42 +106,37 @@ def get_datasets_dict(self) -> dict: """Variable specific args for instrument.""" return { "o2data": { - "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "physical": False, "variables": ["o2"], "output_filename": f"{self.name}_o2.nc", }, "chlorodata": { - "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "physical": False, "variables": ["chl"], "output_filename": f"{self.name}_chl.nc", }, "nitratedata": { - "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "physical": False, "variables": ["no3"], "output_filename": f"{self.name}_no3.nc", }, "phosphatedata": { - "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "physical": False, "variables": ["po4"], "output_filename": f"{self.name}_po4.nc", }, "phdata": { - "dataset_id": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", + "physical": False, "variables": ["ph"], "output_filename": f"{self.name}_ph.nc", }, "phytoplanktondata": { - "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "physical": False, "variables": ["phyc"], "output_filename": f"{self.name}_phyc.nc", }, - "zooplanktondata": { - "dataset_id": "cmems_mod_glo_bgc-plankton_anfc_0.25deg_P1D-m", - "variables": ["zooc"], - "output_filename": f"{self.name}_zooc.nc", - }, "primaryproductiondata": { - "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "physical": False, "variables": ["nppv"], "output_filename": f"{self.name}_nppv.nc", }, @@ -152,17 +147,16 @@ def get_datasets_dict(self) -> dict: class CTD_BGCInstrument(Instrument): """CTD_BGC instrument class.""" - def __init__(self, name, expedition, directory): + def __init__(self, expedition, directory): """Initialize CTD_BGCInstrument.""" filenames = { - "o2": directory.joinpath(f"{name}_o2.nc"), - "chl": directory.joinpath(f"{name}_chl.nc"), - "no3": directory.joinpath(f"{name}_no3.nc"), - "po4": directory.joinpath(f"{name}_po4.nc"), - "ph": directory.joinpath(f"{name}_ph.nc"), - "phyc": directory.joinpath(f"{name}_phyc.nc"), - "zooc": directory.joinpath(f"{name}_zooc.nc"), - "nppv": directory.joinpath(f"{name}_nppv.nc"), + "o2": f"{CTD_BGC.name}_o2.nc", + "chl": f"{CTD_BGC.name}_chl.nc", + "no3": f"{CTD_BGC.name}_no3.nc", + "po4": f"{CTD_BGC.name}_po4.nc", + "ph": f"{CTD_BGC.name}_ph.nc", + "phyc": f"{CTD_BGC.name}_phyc.nc", + "nppv": f"{CTD_BGC.name}_nppv.nc", } variables = { "o2": "o2", @@ -171,7 +165,6 @@ def __init__(self, name, expedition, directory): "po4": "po4", "ph": "ph", "phyc": "phyc", - "zooc": "zooc", "nppv": "nppv", } super().__init__( @@ -184,13 +177,13 @@ def __init__(self, name, expedition, directory): allow_time_extrapolation=True, ) - def simulate(self) -> None: + def simulate(self, measurements, out_path) -> None: """Simulate BGC CTD measurements using Parcels.""" WINCH_SPEED = 1.0 # sink and rise speed in m/s DT = 10.0 # dt of CTD_BGC simulation integrator OUTPUT_DT = timedelta(seconds=10) # output dt for CTD_BGC simulation - if len(self.measurements) == 0: + if len(measurements) == 0: print( "No BGC CTDs provided. Parcels currently crashes when providing an empty particle set, so no BGC CTD simulation will be done and no files will be created." ) @@ -206,7 +199,7 @@ def simulate(self) -> None: if not all( [ np.datetime64(ctd_bgc.spacetime.time) >= fieldset_starttime - for ctd_bgc in self.measurements + for ctd_bgc in measurements ] ): raise ValueError("BGC CTD deployed before fieldset starts.") @@ -246,9 +239,7 @@ def simulate(self) -> None: ) # define output file for the simulation - out_file = ctd_bgc_particleset.ParticleFile( - name=self.out_path, outputdt=OUTPUT_DT - ) + out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=OUTPUT_DT) # execute simulation ctd_bgc_particleset.execute( diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 72db065e..93cd2631 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -70,12 +70,12 @@ def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["uo", "vo"], "output_filename": f"{self.name}_uv.nc", }, "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["thetao"], "output_filename": f"{self.name}_t.nc", }, @@ -86,13 +86,14 @@ def get_datasets_dict(self) -> dict: class DrifterInstrument(Instrument): """Drifter instrument class.""" - def __init__(self, name, expedition, directory): + def __init__(self, expedition, directory): """Initialize DrifterInstrument.""" filenames = { - "UV": directory.joinpath(f"{name}_uv.nc"), - "T": directory.joinpath(f"{name}_t.nc"), + "U": f"{Drifter.name}_uv.nc", + "V": f"{Drifter.name}_uv.nc", + "T": f"{Drifter.name}_t.nc", } - variables = {"UV": ["uo", "vo"], "T": "thetao"} + variables = {"U": "uo", "V": "vo", "T": "thetao"} super().__init__( Drifter.name, expedition, @@ -103,13 +104,13 @@ def __init__(self, name, expedition, directory): allow_time_extrapolation=False, ) - def simulate(self) -> None: + def simulate(self, measurements, out_path) -> None: """Simulate Drifter measurements.""" OUTPUT_DT = timedelta(hours=5) DT = timedelta(minutes=5) ENDTIME = None - if len(self.measurements) == 0: + if len(measurements) == 0: print( "No drifters provided. Parcels currently crashes when providing an empty particle set, so no drifter simulation will be done and no files will be created." ) @@ -122,23 +123,22 @@ def simulate(self) -> None: drifter_particleset = ParticleSet( fieldset=fieldset, pclass=_DrifterParticle, - lat=[drifter.spacetime.location.lat for drifter in self.measurements], - lon=[drifter.spacetime.location.lon for drifter in self.measurements], - depth=[drifter.depth for drifter in self.measurements], - time=[drifter.spacetime.time for drifter in self.measurements], + lat=[drifter.spacetime.location.lat for drifter in measurements], + lon=[drifter.spacetime.location.lon for drifter in measurements], + depth=[drifter.depth for drifter in measurements], + time=[drifter.spacetime.time for drifter in measurements], has_lifetime=[ - 1 if drifter.lifetime is not None else 0 - for drifter in self.measurements + 1 if drifter.lifetime is not None else 0 for drifter in measurements ], lifetime=[ 0 if drifter.lifetime is None else drifter.lifetime.total_seconds() - for drifter in self.measurements + for drifter in measurements ], ) # define output file for the simulation out_file = drifter_particleset.ParticleFile( - name=self.out_path, + name=out_path, outputdt=OUTPUT_DT, chunks=[len(drifter_particleset), 100], ) diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 9c0fc401..4160f474 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -62,12 +62,12 @@ def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { "Sdata": { - "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["so"], "output_filename": f"{self.name}_s.nc", }, "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["thetao"], "output_filename": f"{self.name}_t.nc", }, @@ -78,11 +78,11 @@ def get_datasets_dict(self) -> dict: class Underwater_STInstrument(Instrument): """Underwater_ST instrument class.""" - def __init__(self, name, expedition, directory): + def __init__(self, expedition, directory): """Initialize Underwater_STInstrument.""" filenames = { - "S": directory.joinpath(f"{name}_s.nc"), - "T": directory.joinpath(f"{name}_t.nc"), + "S": f"{Underwater_ST.name}_s.nc", + "T": f"{Underwater_ST.name}_t.nc", } variables = {"S": "so", "T": "thetao"} super().__init__( @@ -95,14 +95,13 @@ def __init__(self, name, expedition, directory): allow_time_extrapolation=True, ) - def simulate(self) -> None: + def simulate(self, measurements, out_path) -> None: """Simulate underway salinity and temperature measurements.""" DEPTH = -2.0 - self.measurements.sort(key=lambda p: p.time) + measurements.sort(key=lambda p: p.time) fieldset = self.load_input_data() - particleset = ParticleSet.from_list( fieldset=fieldset, pclass=_ShipSTParticle, @@ -112,9 +111,9 @@ def simulate(self) -> None: time=0, ) - out_file = particleset.ParticleFile(name=self.out_path, outputdt=np.inf) + out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) - for point in self.measurements: + for point in measurements: particleset.lon_nextloop[:] = point.location.lon particleset.lat_nextloop[:] = point.location.lat particleset.time_nextloop[:] = fieldset.time_origin.reltime( diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 7c916d98..918ef9a2 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -84,17 +84,17 @@ def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["uo", "vo"], "output_filename": f"{self.name}_uv.nc", }, "Sdata": { - "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["so"], "output_filename": f"{self.name}_s.nc", }, "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["thetao"], "output_filename": f"{self.name}_t.nc", }, @@ -105,14 +105,15 @@ def get_datasets_dict(self) -> dict: class XBTInstrument(Instrument): """XBT instrument class.""" - def __init__(self, name, expedition, directory): + def __init__(self, expedition, directory): """Initialize XBTInstrument.""" filenames = { - "UV": directory.joinpath(f"{name}_uv.nc"), - "S": directory.joinpath(f"{name}_s.nc"), - "T": directory.joinpath(f"{name}_t.nc"), + "U": f"{XBT.name}_uv.nc", + "V": f"{XBT.name}_uv.nc", + "S": f"{XBT.name}_s.nc", + "T": f"{XBT.name}_t.nc", } - variables = {"UV": ["uo", "vo"], "S": "so", "T": "thetao"} + variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} super().__init__( XBT.name, expedition, @@ -123,12 +124,12 @@ def __init__(self, name, expedition, directory): allow_time_extrapolation=True, ) - def simulate(self) -> None: + def simulate(self, measurements, out_path) -> None: """Simulate XBT measurements.""" DT = 10.0 # dt of XBT simulation integrator OUTPUT_DT = timedelta(seconds=1) - if len(self.measurements) == 0: + if len(measurements) == 0: print( "No XBTs provided. Parcels currently crashes when providing an empty particle set, so no XBT simulation will be done and no files will be created." ) @@ -144,7 +145,7 @@ def simulate(self) -> None: if not all( [ np.datetime64(xbt.spacetime.time) >= fieldset_starttime - for xbt in self.measurements + for xbt in measurements ] ): raise ValueError("XBT deployed before fieldset starts.") @@ -160,7 +161,7 @@ def simulate(self) -> None: time=0, ), ) - for xbt in self.measurements + for xbt in measurements ] # initial fall speeds @@ -186,7 +187,7 @@ def simulate(self) -> None: fall_speed=[xbt.fall_speed for xbt in self.measurements], ) - out_file = xbt_particleset.ParticleFile(name=self.out_path, outputdt=OUTPUT_DT) + out_file = xbt_particleset.ParticleFile(name=out_path, outputdt=OUTPUT_DT) xbt_particleset.execute( [_sample_temperature, _xbt_cast], diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 8888f657..c70de855 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -62,7 +62,7 @@ def get_instruments(self) -> set[InstrumentType]: instruments_in_expedition.append(InstrumentType.ADCP) if self.instruments_config.ship_underwater_st_config is not None: instruments_in_expedition.append(InstrumentType.UNDERWATER_ST) - return set(instruments_in_expedition) + return sorted(set(instruments_in_expedition), key=lambda x: x.name) except Exception as e: raise InstrumentsConfigError( "Underway instrument config attribute(s) are missing from YAML. Must be Config object or None." @@ -90,7 +90,8 @@ class Schedule(pydantic.BaseModel): def verify( self, ship_speed: float, - input_dir: str | Path | None, + data_dir: str | Path | None, + ignore_missing_bathymetry: bool = False, *, check_space_time_region: bool = False, ) -> None: @@ -103,14 +104,6 @@ def verify( 3. Waypoint times are in ascending order. 4. All waypoints are in water (not on land). 5. The ship can arrive on time at each waypoint given its speed. - - :param ship_speed: The ship's speed in knots. - :param input_dir: The input directory containing necessary files. - :param check_space_time_region: whether to check for missing space_time_region. - # :param ignore_missing_fieldsets: whether to ignore warning for missing field sets. - :raises PlanningError: If any of the verification checks fail, indicating infeasible or incorrect waypoints. - :raises NotImplementedError: If an instrument in the schedule is not implemented. - :return: None. The method doesn't return a value but raises exceptions if verification fails. """ print("\nVerifying route... ") @@ -139,18 +132,20 @@ def verify( # check if all waypoints are in water using bathymetry data # TODO: tests should be updated to check this! + # TODO: write test that checks that will flag when waypoint is on land!! [add to existing suite of fail .verify() tests in test_expedition.py] + # TODO: need to do an overhaul of the DATA which is in tests/expedition/expedition_dir - don't think it's currently suitable! land_waypoints = [] - if input_dir is not None: - bathymetry_path = input_dir.joinpath("bathymetry.nc") + if data_dir is not None: + bathymetry_path = data_dir.joinpath("bathymetry.nc") try: bathymetry_field = Field.from_netcdf( bathymetry_path, - variables=("bathymetry", "deptho"), + variable=("bathymetry", "deptho"), dimensions={"lon": "longitude", "lat": "latitude"}, ) except Exception as e: - raise FileNotFoundError( - "Bathymetry file not found in input data. Cannot verify waypoints are in water." + raise ScheduleError( + f"Problem loading bathymetry data (used to verify waypoints are in water): {e}" ) from e for wp_i, wp in enumerate(self.waypoints): bathy = bathymetry_field.eval( @@ -159,12 +154,17 @@ def verify( wp.location.lat, wp.location.lon, ) - if np.isnan(bathy) or bathy >= 0: + if np.isnan(bathy) or bathy <= 0: land_waypoints.append((wp_i, wp)) + if len(land_waypoints) > 0: raise ScheduleError( f"The following waypoints are on land: {['#' + str(wp_i + 1) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" ) + elif not ignore_missing_bathymetry: + raise ScheduleError( + "Cannot verify waypoints are in water as bathymetry data not found. Have you run `virtualship fetch` command?" + ) # check that ship will arrive on time at each waypoint (in case no unexpected event happen) time = self.waypoints[0].time From 1081fab71eb481d665a8f66c1d8dea3c09b72d53 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 29 Oct 2025 13:01:08 +0100 Subject: [PATCH 33/97] adding U and V to instruments where missing --- src/virtualship/instruments/base.py | 19 ++++++------ src/virtualship/instruments/ctd.py | 29 +++++++++++++------ src/virtualship/instruments/ctd_bgc.py | 17 +++++++++-- .../instruments/ship_underwater_st.py | 10 ++++++- 4 files changed, 54 insertions(+), 21 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 6ae38cc0..c45c71fe 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -4,13 +4,11 @@ import copernicusmarine import numpy as np -from yaspin import yaspin from parcels import Field, FieldSet from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash from virtualship.errors import CopernicusCatalogueError from virtualship.models import Expedition, SpaceTimeRegion -from virtualship.utils import ship_spinner PRODUCT_IDS = { "phys": { @@ -48,6 +46,9 @@ class InputDataset(abc.ABC): """Base class for instrument input datasets.""" + # TODO: data download is performed per instrument (in `fetch`), which is a bit inefficient when some instruments can share dataa. + # TODO: However, future changes, with Parcels-v4 and copernicusmarine direct ingestion, will hopefully remove the need for fetch. + def __init__( self, name: str, @@ -288,13 +289,13 @@ def run(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" # TODO: this will have to be able to handle the non-spinner/instead progress bar for drifters and argos! - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: - self.simulate(measurements, out_path) - spinner.ok("✅") + # with yaspin( + # text=f"Simulating {self.name} measurements... ", + # side="right", + # spinner=ship_spinner, + # ) as spinner: + self.simulate(measurements, out_path) + # spinner.ok("✅") def _get_data_dir(self, expedition_dir: Path) -> Path: space_time_region_hash = get_space_time_region_hash( diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index c1981e06..ffc88922 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -80,6 +80,11 @@ def __init__(self, data_dir, credentials, space_time_region): def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { + "UVdata": { + "physical": True, # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? + "variables": ["uo", "vo"], + "output_filename": f"{self.name}_uv.nc", + }, "Sdata": { "physical": True, "variables": ["so"], @@ -100,10 +105,12 @@ class CTDInstrument(Instrument): def __init__(self, expedition, directory): """Initialize CTDInstrument.""" filenames = { + "U": f"{CTD.name}_uv.nc", # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? + "V": f"{CTD.name}_uv.nc", "S": f"{CTD.name}_s.nc", "T": f"{CTD.name}_t.nc", } - variables = {"S": "so", "T": "thetao"} + variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} super().__init__( CTD.name, @@ -130,8 +137,12 @@ def simulate(self, measurements, out_path) -> None: fieldset = self.load_input_data() - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + fieldset_starttime = fieldset.T.grid.time_origin.fulltime( + fieldset.T.grid.time_full[0] + ) + fieldset_endtime = fieldset.T.grid.time_origin.fulltime( + fieldset.T.grid.time_full[-1] + ) # deploy time for all ctds should be later than fieldset start time if not all( @@ -167,13 +178,13 @@ def simulate(self, measurements, out_path) -> None: ctd_particleset = ParticleSet( fieldset=fieldset, pclass=_CTDParticle, - lon=[ctd.spacetime.location.lon for ctd in self.measurements], - lat=[ctd.spacetime.location.lat for ctd in self.measurements], - depth=[ctd.min_depth for ctd in self.measurements], - time=[ctd.spacetime.time for ctd in self.measurements], + lon=[ctd.spacetime.location.lon for ctd in measurements], + lat=[ctd.spacetime.location.lat for ctd in measurements], + depth=[ctd.min_depth for ctd in measurements], + time=[ctd.spacetime.time for ctd in measurements], max_depth=max_depths, - min_depth=[ctd.min_depth for ctd in self.measurements], - winch_speed=[WINCH_SPEED for _ in self.measurements], + min_depth=[ctd.min_depth for ctd in measurements], + winch_speed=[WINCH_SPEED for _ in measurements], ) # define output file for the simulation diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index ed9ab9d9..1b702b6e 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -105,6 +105,11 @@ def __init__(self, data_dir, credentials, space_time_region): def get_datasets_dict(self) -> dict: """Variable specific args for instrument.""" return { + "UVdata": { + "physical": True, # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? + "variables": ["uo", "vo"], + "output_filename": f"{self.name}_uv.nc", + }, "o2data": { "physical": False, "variables": ["o2"], @@ -150,6 +155,8 @@ class CTD_BGCInstrument(Instrument): def __init__(self, expedition, directory): """Initialize CTD_BGCInstrument.""" filenames = { + "U": f"{CTD_BGC.name}_uv.nc", # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? + "V": f"{CTD_BGC.name}_uv.nc", "o2": f"{CTD_BGC.name}_o2.nc", "chl": f"{CTD_BGC.name}_chl.nc", "no3": f"{CTD_BGC.name}_no3.nc", @@ -159,6 +166,8 @@ def __init__(self, expedition, directory): "nppv": f"{CTD_BGC.name}_nppv.nc", } variables = { + "U": "uo", + "V": "vo", "o2": "o2", "chl": "chl", "no3": "no3", @@ -192,8 +201,12 @@ def simulate(self, measurements, out_path) -> None: fieldset = self.load_input_data() - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + fieldset_starttime = fieldset.o2.grid.time_origin.fulltime( + fieldset.o2.grid.time_full[0] + ) + fieldset_endtime = fieldset.o2.grid.time_origin.fulltime( + fieldset.o2.grid.time_full[-1] + ) # deploy time for all ctds should be later than fieldset start time if not all( diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 4160f474..2ffbd762 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -61,6 +61,11 @@ def __init__(self, data_dir, credentials, space_time_region): def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { + "UVdata": { + "physical": True, # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? + "variables": ["uo", "vo"], + "output_filename": f"{self.name}_uv.nc", + }, "Sdata": { "physical": True, "variables": ["so"], @@ -81,10 +86,13 @@ class Underwater_STInstrument(Instrument): def __init__(self, expedition, directory): """Initialize Underwater_STInstrument.""" filenames = { + "U": f"{Underwater_ST.name}_uv.nc", # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? + "V": f"{Underwater_ST.name}_uv.nc", "S": f"{Underwater_ST.name}_s.nc", "T": f"{Underwater_ST.name}_t.nc", } - variables = {"S": "so", "T": "thetao"} + variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} + super().__init__( Underwater_ST.name, expedition, From a0d7d2c64d033c2ac71c171a58c59aba3c61a4fb Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 29 Oct 2025 13:07:43 +0100 Subject: [PATCH 34/97] enhanced error messaging for XBT in too shallow regions --- src/virtualship/instruments/xbt.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 918ef9a2..b15c0e6f 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -165,26 +165,26 @@ def simulate(self, measurements, out_path) -> None: ] # initial fall speeds - initial_fall_speeds = [xbt.fall_speed for xbt in self.measurements] + initial_fall_speeds = [xbt.fall_speed for xbt in measurements] # XBT depth can not be too shallow, because kernel would break. for max_depth, fall_speed in zip(max_depths, initial_fall_speeds, strict=False): if not max_depth <= -DT * fall_speed: raise ValueError( - f"XBT max_depth or bathymetry shallower than maximum {-DT * fall_speed}" + f"XBT max_depth or bathymetry shallower than minimum {-DT * fall_speed}. It is likely the XBT cannot be deployed in this area, which is too shallow." ) # define xbt particles xbt_particleset = ParticleSet( fieldset=fieldset, pclass=_XBTParticle, - lon=[xbt.spacetime.location.lon for xbt in self.measurements], - lat=[xbt.spacetime.location.lat for xbt in self.measurements], - depth=[xbt.min_depth for xbt in self.measurements], - time=[xbt.spacetime.time for xbt in self.measurements], + lon=[xbt.spacetime.location.lon for xbt in measurements], + lat=[xbt.spacetime.location.lat for xbt in measurements], + depth=[xbt.min_depth for xbt in measurements], + time=[xbt.spacetime.time for xbt in measurements], max_depth=max_depths, - min_depth=[xbt.min_depth for xbt in self.measurements], - fall_speed=[xbt.fall_speed for xbt in self.measurements], + min_depth=[xbt.min_depth for xbt in measurements], + fall_speed=[xbt.fall_speed for xbt in measurements], ) out_file = xbt_particleset.ParticleFile(name=out_path, outputdt=OUTPUT_DT) From e5c08ce049be8440d7039cb2eb2e9af7f79ba59a Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 29 Oct 2025 13:07:55 +0100 Subject: [PATCH 35/97] bug fixes --- src/virtualship/instruments/ctd_bgc.py | 14 +++++++------- src/virtualship/instruments/drifter.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 1b702b6e..f3174fd9 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -228,7 +228,7 @@ def simulate(self, measurements, out_path) -> None: time=0, ), ) - for ctd_bgc in self.measurements + for ctd_bgc in measurements ] # CTD depth can not be too shallow, because kernel would break. @@ -242,13 +242,13 @@ def simulate(self, measurements, out_path) -> None: ctd_bgc_particleset = ParticleSet( fieldset=fieldset, pclass=_CTD_BGCParticle, - lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in self.measurements], - lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in self.measurements], - depth=[ctd_bgc.min_depth for ctd_bgc in self.measurements], - time=[ctd_bgc.spacetime.time for ctd_bgc in self.measurements], + lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in measurements], + lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in measurements], + depth=[ctd_bgc.min_depth for ctd_bgc in measurements], + time=[ctd_bgc.spacetime.time for ctd_bgc in measurements], max_depth=max_depths, - min_depth=[ctd_bgc.min_depth for ctd_bgc in self.measurements], - winch_speed=[WINCH_SPEED for _ in self.measurements], + min_depth=[ctd_bgc.min_depth for ctd_bgc in measurements], + winch_speed=[WINCH_SPEED for _ in measurements], ) # define output file for the simulation diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 93cd2631..f33fde55 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -164,7 +164,7 @@ def simulate(self, measurements, out_path) -> None: # if there are more particles left than the number of drifters with an indefinite endtime, warn the user if len(drifter_particleset.particledata) > len( - [d for d in self.measurements if d.lifetime is None] + [d for d in measurements if d.lifetime is None] ): print( "WARN: Some drifters had a life time beyond the end time of the fieldset or the requested end time." From a4f8af0c88f65837eccd82fd420c7904cd8962f3 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 08:43:52 +0100 Subject: [PATCH 36/97] version U and V downloaded --- src/virtualship/instruments/base.py | 2 ++ src/virtualship/instruments/ctd.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index c45c71fe..23b324e5 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -245,6 +245,8 @@ def load_input_data(self) -> FieldSet: # TODO: can simulate_schedule.py be refactored to be contained in base.py and repsective instrument files too...? # TODO: tests need updating...! + #! TODO: CTD, CTD_BGC and Underway_ST deployment testing in `run` is outstanding! + try: data_dir = self._get_data_dir(self.directory) joined_filepaths = { diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index ffc88922..86c6e0ac 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -10,6 +10,9 @@ from virtualship.models import Spacetime from virtualship.utils import register_input_dataset, register_instrument +# TODO: add some kind of check that each instrument has a dataclass, particle class, InputDataset class and Instrument class? +# TODO: probably as a test + @dataclass class CTD: @@ -33,6 +36,9 @@ class CTD: ) +# TODO: way to group kernels together, just to make clearer? + + def _sample_temperature(particle, fieldset, time): particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] @@ -104,6 +110,8 @@ class CTDInstrument(Instrument): def __init__(self, expedition, directory): """Initialize CTDInstrument.""" + #! TODO: actually don't need to download U and V for CTD simulation... can instead add mock/duplicate of T and name it U (also don't need V)! + filenames = { "U": f"{CTD.name}_uv.nc", # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? "V": f"{CTD.name}_uv.nc", From e3c57f628586b89344734b276ca121f668b1fcdd Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 10:30:09 +0100 Subject: [PATCH 37/97] dummy U and V --- src/virtualship/instruments/base.py | 19 +++++++------- src/virtualship/instruments/ctd.py | 14 ++++------- src/virtualship/instruments/ctd_bgc.py | 14 +++-------- .../instruments/ship_underwater_st.py | 15 +++++------ src/virtualship/utils.py | 25 +++++++++++++++++++ 5 files changed, 49 insertions(+), 38 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 23b324e5..6c036b28 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -4,11 +4,13 @@ import copernicusmarine import numpy as np +from yaspin import yaspin from parcels import Field, FieldSet from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash from virtualship.errors import CopernicusCatalogueError from virtualship.models import Expedition, SpaceTimeRegion +from virtualship.utils import ship_spinner PRODUCT_IDS = { "phys": { @@ -244,9 +246,6 @@ def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" # TODO: can simulate_schedule.py be refactored to be contained in base.py and repsective instrument files too...? # TODO: tests need updating...! - - #! TODO: CTD, CTD_BGC and Underway_ST deployment testing in `run` is outstanding! - try: data_dir = self._get_data_dir(self.directory) joined_filepaths = { @@ -291,13 +290,13 @@ def run(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" # TODO: this will have to be able to handle the non-spinner/instead progress bar for drifters and argos! - # with yaspin( - # text=f"Simulating {self.name} measurements... ", - # side="right", - # spinner=ship_spinner, - # ) as spinner: - self.simulate(measurements, out_path) - # spinner.ok("✅") + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate(measurements, out_path) + spinner.ok("✅") def _get_data_dir(self, expedition_dir: Path) -> Path: space_time_region_hash = get_space_time_region_hash( diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 86c6e0ac..22c69799 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -8,7 +8,7 @@ from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models import Spacetime -from virtualship.utils import register_input_dataset, register_instrument +from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument # TODO: add some kind of check that each instrument has a dataclass, particle class, InputDataset class and Instrument class? # TODO: probably as a test @@ -86,11 +86,6 @@ def __init__(self, data_dir, credentials, space_time_region): def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { - "UVdata": { - "physical": True, # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? - "variables": ["uo", "vo"], - "output_filename": f"{self.name}_uv.nc", - }, "Sdata": { "physical": True, "variables": ["so"], @@ -113,12 +108,10 @@ def __init__(self, expedition, directory): #! TODO: actually don't need to download U and V for CTD simulation... can instead add mock/duplicate of T and name it U (also don't need V)! filenames = { - "U": f"{CTD.name}_uv.nc", # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? - "V": f"{CTD.name}_uv.nc", "S": f"{CTD.name}_s.nc", "T": f"{CTD.name}_t.nc", } - variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} + variables = {"S": "so", "T": "thetao"} super().__init__( CTD.name, @@ -145,6 +138,9 @@ def simulate(self, measurements, out_path) -> None: fieldset = self.load_input_data() + # add dummy U + add_dummy_UV(fieldset) # TODO: parcels v3 bodge; remove when parcels v4 is used + fieldset_starttime = fieldset.T.grid.time_origin.fulltime( fieldset.T.grid.time_full[0] ) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index f3174fd9..c4245823 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -8,7 +8,7 @@ from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_input_dataset, register_instrument +from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument @dataclass @@ -105,11 +105,6 @@ def __init__(self, data_dir, credentials, space_time_region): def get_datasets_dict(self) -> dict: """Variable specific args for instrument.""" return { - "UVdata": { - "physical": True, # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? - "variables": ["uo", "vo"], - "output_filename": f"{self.name}_uv.nc", - }, "o2data": { "physical": False, "variables": ["o2"], @@ -155,8 +150,6 @@ class CTD_BGCInstrument(Instrument): def __init__(self, expedition, directory): """Initialize CTD_BGCInstrument.""" filenames = { - "U": f"{CTD_BGC.name}_uv.nc", # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? - "V": f"{CTD_BGC.name}_uv.nc", "o2": f"{CTD_BGC.name}_o2.nc", "chl": f"{CTD_BGC.name}_chl.nc", "no3": f"{CTD_BGC.name}_no3.nc", @@ -166,8 +159,6 @@ def __init__(self, expedition, directory): "nppv": f"{CTD_BGC.name}_nppv.nc", } variables = { - "U": "uo", - "V": "vo", "o2": "o2", "chl": "chl", "no3": "no3", @@ -201,6 +192,9 @@ def simulate(self, measurements, out_path) -> None: fieldset = self.load_input_data() + # add dummy U + add_dummy_UV(fieldset) # TODO: parcels v3 bodge; remove when parcels v4 is used + fieldset_starttime = fieldset.o2.grid.time_origin.fulltime( fieldset.o2.grid.time_full[0] ) diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 2ffbd762..f2393778 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -6,7 +6,7 @@ from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType -from virtualship.utils import register_input_dataset, register_instrument +from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument @dataclass @@ -61,11 +61,6 @@ def __init__(self, data_dir, credentials, space_time_region): def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { - "UVdata": { - "physical": True, # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? - "variables": ["uo", "vo"], - "output_filename": f"{self.name}_uv.nc", - }, "Sdata": { "physical": True, "variables": ["so"], @@ -86,12 +81,10 @@ class Underwater_STInstrument(Instrument): def __init__(self, expedition, directory): """Initialize Underwater_STInstrument.""" filenames = { - "U": f"{Underwater_ST.name}_uv.nc", # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? - "V": f"{Underwater_ST.name}_uv.nc", "S": f"{Underwater_ST.name}_s.nc", "T": f"{Underwater_ST.name}_t.nc", } - variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} + variables = {"S": "so", "T": "thetao"} super().__init__( Underwater_ST.name, @@ -110,6 +103,10 @@ def simulate(self, measurements, out_path) -> None: measurements.sort(key=lambda p: p.time) fieldset = self.load_input_data() + + # add dummy U + add_dummy_UV(fieldset) # TODO: parcels v3 bodge; remove when parcels v4 is used + particleset = ParticleSet.from_list( fieldset=fieldset, pclass=_ShipSTParticle, diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 6ffaefe7..324f8c67 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -8,6 +8,8 @@ from pathlib import Path from typing import TYPE_CHECKING, TextIO +from parcels import FieldSet + if TYPE_CHECKING: from virtualship.models import Expedition @@ -268,3 +270,26 @@ def get_input_dataset_class(instrument_type): def get_instrument_class(instrument_type): return INSTRUMENT_CLASS_MAP.get(instrument_type) + + +def add_dummy_UV(fieldset: FieldSet): + """Add a dummy U and V field to a FieldSet to satisfy parcels FieldSet completeness checks.""" + if "U" not in fieldset.__dict__.keys(): + for uv_var in ["U", "V"]: + dummy_field = getattr( + FieldSet.from_data( + {"U": 0, "V": 0}, {"lon": 0, "lat": 0}, mesh="spherical" + ), + uv_var, + ) + fieldset.add_field(dummy_field) + try: + fieldset.time_origin = ( + fieldset.T.grid.time_origin + if "T" in fieldset.__dict__.keys() + else fieldset.o2.grid.time_origin + ) + except Exception: + raise ValueError( + "Cannot determine time_origin for dummy UV fields. Assert T or o2 exists in fieldset." + ) from None From 813a245a159547db42cb8bce58f7e36e1ac55076 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 11:22:33 +0100 Subject: [PATCH 38/97] Neaten up logging output --- src/virtualship/expedition/do_expedition.py | 6 +++++ src/virtualship/instruments/adcp.py | 3 ++- src/virtualship/instruments/argo_float.py | 4 +++- src/virtualship/instruments/base.py | 22 ++++++++++++++----- src/virtualship/instruments/ctd.py | 3 ++- src/virtualship/instruments/ctd_bgc.py | 4 +++- src/virtualship/instruments/drifter.py | 3 ++- .../instruments/ship_underwater_st.py | 3 ++- src/virtualship/instruments/xbt.py | 3 ++- 9 files changed, 38 insertions(+), 13 deletions(-) diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index ec61b1b0..b857b72c 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -1,5 +1,6 @@ """do_expedition function.""" +import logging import os import shutil from pathlib import Path @@ -26,6 +27,11 @@ projection = pyproj.Geod(ellps="WGS84") +# parcels logger (suppress INFO messages to prevent log being flooded) +external_logger = logging.getLogger("parcels.tools.loggers") +external_logger.setLevel(logging.WARNING) + + def do_expedition(expedition_dir: str | Path) -> None: """ Perform an expedition, providing terminal feedback and file output. diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index db5be8ef..20a12436 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -89,6 +89,7 @@ def __init__(self, expedition, directory): variables, add_bathymetry=False, allow_time_extrapolation=True, + verbose_progress=False, ) def simulate(self, measurements, out_path) -> None: @@ -127,6 +128,6 @@ def simulate(self, measurements, out_path) -> None: [_sample_velocity], dt=1, runtime=1, - verbose_progress=False, + verbose_progress=self.verbose_progress, output_file=out_file, ) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 93aa9093..898fd5cc 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -138,6 +138,7 @@ def __init__(self, data_dir, credentials, space_time_region): data_dir, credentials, space_time_region, + verbose_progress=True, ) def get_datasets_dict(self) -> dict: @@ -182,6 +183,7 @@ def __init__(self, expedition, directory): variables, add_bathymetry=False, allow_time_extrapolation=False, + verbose_progress=True, ) def simulate(self, measurements, out_path) -> None: @@ -243,5 +245,5 @@ def simulate(self, measurements, out_path) -> None: endtime=actual_endtime, dt=DT, output_file=out_file, - verbose_progress=True, + verbose_progress=self.verbose_progress, ) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 6c036b28..8212aab7 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -224,6 +224,7 @@ def __init__( variables: dict, add_bathymetry: bool, allow_time_extrapolation: bool, + verbose_progress: bool, bathymetry_file: str = "bathymetry.nc", ): """Initialise instrument.""" @@ -241,11 +242,16 @@ def __init__( self.bathymetry_file = bathymetry_file self.add_bathymetry = add_bathymetry self.allow_time_extrapolation = allow_time_extrapolation + self.verbose_progress = verbose_progress def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" # TODO: can simulate_schedule.py be refactored to be contained in base.py and repsective instrument files too...? # TODO: tests need updating...! + + #! TODO: E.g. ADCP is giving too much depth data?! + #! TODO: in fact output from most instruments doesn't look quite right...? + try: data_dir = self._get_data_dir(self.directory) joined_filepaths = { @@ -290,13 +296,17 @@ def run(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" # TODO: this will have to be able to handle the non-spinner/instead progress bar for drifters and argos! - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate(measurements, out_path) + spinner.ok("✅") + else: + print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) - spinner.ok("✅") def _get_data_dir(self, expedition_dir: Path) -> Path: space_time_region_hash = get_space_time_region_hash( diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 22c69799..9b59a770 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -121,6 +121,7 @@ def __init__(self, expedition, directory): variables, add_bathymetry=True, allow_time_extrapolation=True, + verbose_progress=False, ) def simulate(self, measurements, out_path) -> None: @@ -199,7 +200,7 @@ def simulate(self, measurements, out_path) -> None: [_sample_salinity, _sample_temperature, _ctd_cast], endtime=fieldset_endtime, dt=DT, - verbose_progress=False, + verbose_progress=self.verbose_progress, output_file=out_file, ) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index c4245823..397f1af4 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -100,6 +100,7 @@ def __init__(self, data_dir, credentials, space_time_region): data_dir, credentials, space_time_region, + verbose_progress=False, ) def get_datasets_dict(self) -> dict: @@ -175,6 +176,7 @@ def __init__(self, expedition, directory): variables, add_bathymetry=True, allow_time_extrapolation=True, + verbose_progress=False, ) def simulate(self, measurements, out_path) -> None: @@ -262,7 +264,7 @@ def simulate(self, measurements, out_path) -> None: ], endtime=fieldset_endtime, dt=DT, - verbose_progress=False, + verbose_progress=self.verbose_progress, output_file=out_file, ) diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index f33fde55..3b047fed 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -102,6 +102,7 @@ def __init__(self, expedition, directory): variables, add_bathymetry=False, allow_time_extrapolation=False, + verbose_progress=True, ) def simulate(self, measurements, out_path) -> None: @@ -159,7 +160,7 @@ def simulate(self, measurements, out_path) -> None: endtime=actual_endtime, dt=DT, output_file=out_file, - verbose_progress=True, + verbose_progress=self.verbose_progress, ) # if there are more particles left than the number of drifters with an indefinite endtime, warn the user diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index f2393778..b52887a2 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -94,6 +94,7 @@ def __init__(self, expedition, directory): variables, add_bathymetry=False, allow_time_extrapolation=True, + verbose_progress=False, ) def simulate(self, measurements, out_path) -> None: @@ -129,6 +130,6 @@ def simulate(self, measurements, out_path) -> None: [_sample_salinity, _sample_temperature], dt=1, runtime=1, - verbose_progress=False, + verbose_progress=self.verbose_progress, output_file=out_file, ) diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index b15c0e6f..641015f5 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -122,6 +122,7 @@ def __init__(self, expedition, directory): variables, add_bathymetry=True, allow_time_extrapolation=True, + verbose_progress=False, ) def simulate(self, measurements, out_path) -> None: @@ -193,7 +194,7 @@ def simulate(self, measurements, out_path) -> None: [_sample_temperature, _xbt_cast], endtime=fieldset_endtime, dt=DT, - verbose_progress=False, + verbose_progress=self.verbose_progress, output_file=out_file, ) From b94f4f0b1b7c43caf4d72e5f9762494b1366c42b Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 13:14:26 +0100 Subject: [PATCH 39/97] small bug fixes --- src/virtualship/instruments/argo_float.py | 1 - src/virtualship/instruments/ctd_bgc.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 898fd5cc..4a5dcdef 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -138,7 +138,6 @@ def __init__(self, data_dir, credentials, space_time_region): data_dir, credentials, space_time_region, - verbose_progress=True, ) def get_datasets_dict(self) -> dict: diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 397f1af4..537ba810 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -100,7 +100,6 @@ def __init__(self, data_dir, credentials, space_time_region): data_dir, credentials, space_time_region, - verbose_progress=False, ) def get_datasets_dict(self) -> dict: From a5d10e7d4ab9e417c195c460d41c7ba9944b1718 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 13:43:00 +0100 Subject: [PATCH 40/97] tidy up --- src/virtualship/instruments/adcp.py | 25 +++++++++++++++++-- src/virtualship/instruments/argo_float.py | 22 ++++++++++++++++ src/virtualship/instruments/base.py | 7 ++---- src/virtualship/instruments/ctd.py | 24 +++++++++++++++--- src/virtualship/instruments/ctd_bgc.py | 22 ++++++++++++++++ src/virtualship/instruments/drifter.py | 22 ++++++++++++++++ .../instruments/ship_underwater_st.py | 22 ++++++++++++++++ src/virtualship/instruments/xbt.py | 22 ++++++++++++++++ src/virtualship/models/expedition.py | 1 - 9 files changed, 156 insertions(+), 11 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 20a12436..b2ebe988 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -11,6 +11,10 @@ register_instrument, ) +# ===================================================== +# SECTION: Dataclass +# ===================================================== + @dataclass class ADCP: @@ -19,8 +23,11 @@ class ADCP: name: ClassVar[str] = "ADCP" -# we specifically use ScipyParticle because we have many small calls to execute -# there is some overhead with JITParticle and this ends up being significantly faster +# ===================================================== +# SECTION: Particle Class +# ===================================================== + + _ADCPParticle = ScipyParticle.add_variables( [ Variable("U", dtype=np.float32, initial=np.nan), @@ -28,6 +35,10 @@ class ADCP: ] ) +# ===================================================== +# SECTION: Kernels +# ===================================================== + def _sample_velocity(particle, fieldset, time): particle.U, particle.V = fieldset.UV.eval( @@ -35,6 +46,11 @@ def _sample_velocity(particle, fieldset, time): ) +# ===================================================== +# SECTION: InputDataset Class +# ===================================================== + + @register_input_dataset(InstrumentType.ADCP) class ADCPInputDataset(InputDataset): """Input dataset for ADCP instrument.""" @@ -70,6 +86,11 @@ def get_datasets_dict(self) -> dict: } +# ===================================================== +# SECTION: Instrument Class +# ===================================================== + + @register_instrument(InstrumentType.ADCP) class ADCPInstrument(Instrument): """ADCP instrument class.""" diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 4a5dcdef..573836b4 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -17,6 +17,10 @@ from virtualship.models.spacetime import Spacetime from virtualship.utils import register_input_dataset, register_instrument +# ===================================================== +# SECTION: Dataclass +# ===================================================== + @dataclass class ArgoFloat: @@ -32,6 +36,10 @@ class ArgoFloat: drift_days: float +# ===================================================== +# SECTION: Particle Class +# ===================================================== + _ArgoParticle = JITParticle.add_variables( [ Variable("cycle_phase", dtype=np.int32, initial=0.0), @@ -48,6 +56,10 @@ class ArgoFloat: ] ) +# ===================================================== +# SECTION: Kernels +# ===================================================== + def _argo_float_vertical_movement(particle, fieldset, time): if particle.cycle_phase == 0: @@ -116,6 +128,11 @@ def _check_error(particle, fieldset, time): particle.delete() +# ===================================================== +# SECTION: InputDataset Class +# ===================================================== + + @register_input_dataset(InstrumentType.ARGO_FLOAT) class ArgoFloatInputDataset(InputDataset): """Input dataset for ArgoFloat instrument.""" @@ -161,6 +178,11 @@ def get_datasets_dict(self) -> dict: } +# ===================================================== +# SECTION: Instrument Class +# ===================================================== + + @register_instrument(InstrumentType.ARGO_FLOAT) class ArgoFloatInstrument(Instrument): """ArgoFloat instrument class.""" diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 8212aab7..3c6a333b 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -246,12 +246,8 @@ def __init__( def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" - # TODO: can simulate_schedule.py be refactored to be contained in base.py and repsective instrument files too...? # TODO: tests need updating...! - #! TODO: E.g. ADCP is giving too much depth data?! - #! TODO: in fact output from most instruments doesn't look quite right...? - try: data_dir = self._get_data_dir(self.directory) joined_filepaths = { @@ -303,10 +299,11 @@ def run(self, measurements: list, out_path: str | Path) -> None: spinner=ship_spinner, ) as spinner: self.simulate(measurements, out_path) - spinner.ok("✅") + spinner.ok("✅\n") else: print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) + print("\n") def _get_data_dir(self, expedition_dir: Path) -> Path: space_time_region_hash = get_space_time_region_hash( diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 9b59a770..5080c006 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -13,6 +13,10 @@ # TODO: add some kind of check that each instrument has a dataclass, particle class, InputDataset class and Instrument class? # TODO: probably as a test +# ===================================================== +# SECTION: Dataclass +# ===================================================== + @dataclass class CTD: @@ -24,6 +28,10 @@ class CTD: max_depth: float +# ===================================================== +# SECTION: Particle Class +# ===================================================== + _CTDParticle = JITParticle.add_variables( [ Variable("salinity", dtype=np.float32, initial=np.nan), @@ -36,7 +44,9 @@ class CTD: ) -# TODO: way to group kernels together, just to make clearer? +# ===================================================== +# SECTION: Kernels +# ===================================================== def _sample_temperature(particle, fieldset, time): @@ -61,6 +71,11 @@ def _ctd_cast(particle, fieldset, time): particle.delete() +# ===================================================== +# SECTION: InputDataset Class +# ===================================================== + + @register_input_dataset(InstrumentType.CTD) class CTDInputDataset(InputDataset): """Input dataset for CTD instrument.""" @@ -99,14 +114,17 @@ def get_datasets_dict(self) -> dict: } +# ===================================================== +# SECTION: Instrument Class +# ===================================================== + + @register_instrument(InstrumentType.CTD) class CTDInstrument(Instrument): """CTD instrument class.""" def __init__(self, expedition, directory): """Initialize CTDInstrument.""" - #! TODO: actually don't need to download U and V for CTD simulation... can instead add mock/duplicate of T and name it U (also don't need V)! - filenames = { "S": f"{CTD.name}_s.nc", "T": f"{CTD.name}_t.nc", diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 537ba810..28dd55c3 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -10,6 +10,10 @@ from virtualship.models.spacetime import Spacetime from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument +# ===================================================== +# SECTION: Dataclass +# ===================================================== + @dataclass class CTD_BGC: @@ -21,6 +25,10 @@ class CTD_BGC: max_depth: float +# ===================================================== +# SECTION: Particle Class +# ===================================================== + _CTD_BGCParticle = JITParticle.add_variables( [ Variable("o2", dtype=np.float32, initial=np.nan), @@ -37,6 +45,10 @@ class CTD_BGC: ] ) +# ===================================================== +# SECTION: Kernels +# ===================================================== + def _sample_o2(particle, fieldset, time): particle.o2 = fieldset.o2[time, particle.depth, particle.lat, particle.lon] @@ -80,6 +92,11 @@ def _ctd_bgc_cast(particle, fieldset, time): particle.delete() +# ===================================================== +# SECTION: InputDataset Class +# ===================================================== + + @register_input_dataset(InstrumentType.CTD_BGC) class CTD_BGCInputDataset(InputDataset): """Input dataset object for CTD_BGC instrument.""" @@ -143,6 +160,11 @@ def get_datasets_dict(self) -> dict: } +# ===================================================== +# SECTION: Instrument Class +# ===================================================== + + @register_instrument(InstrumentType.CTD_BGC) class CTD_BGCInstrument(Instrument): """CTD_BGC instrument class.""" diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 3b047fed..0ce78624 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -10,6 +10,10 @@ from virtualship.models.spacetime import Spacetime from virtualship.utils import register_input_dataset, register_instrument +# ===================================================== +# SECTION: Dataclass +# ===================================================== + @dataclass class Drifter: @@ -21,6 +25,10 @@ class Drifter: lifetime: timedelta | None # if none, lifetime is infinite +# ===================================================== +# SECTION: Particle Class +# ===================================================== + _DrifterParticle = JITParticle.add_variables( [ Variable("temperature", dtype=np.float32, initial=np.nan), @@ -30,6 +38,10 @@ class Drifter: ] ) +# ===================================================== +# SECTION: Kernels +# ===================================================== + def _sample_temperature(particle, fieldset, time): particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] @@ -42,6 +54,11 @@ def _check_lifetime(particle, fieldset, time): particle.delete() +# ===================================================== +# SECTION: InputDataset Class +# ===================================================== + + @register_input_dataset(InstrumentType.DRIFTER) class DrifterInputDataset(InputDataset): """Input dataset for Drifter instrument.""" @@ -82,6 +99,11 @@ def get_datasets_dict(self) -> dict: } +# ===================================================== +# SECTION: Instrument Class +# ===================================================== + + @register_instrument(InstrumentType.DRIFTER) class DrifterInstrument(Instrument): """Drifter instrument class.""" diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index b52887a2..946e1a9b 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -8,6 +8,10 @@ from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument +# ===================================================== +# SECTION: Dataclass +# ===================================================== + @dataclass class Underwater_ST: @@ -16,6 +20,10 @@ class Underwater_ST: name: ClassVar[str] = "Underwater_ST" +# ===================================================== +# SECTION: Particle Class +# ===================================================== + _ShipSTParticle = ScipyParticle.add_variables( [ Variable("S", dtype=np.float32, initial=np.nan), @@ -23,6 +31,10 @@ class Underwater_ST: ] ) +# ===================================================== +# SECTION: Kernels +# ===================================================== + # define function sampling Salinity def _sample_salinity(particle, fieldset, time): @@ -34,6 +46,11 @@ def _sample_temperature(particle, fieldset, time): particle.T = fieldset.T[time, particle.depth, particle.lat, particle.lon] +# ===================================================== +# SECTION: InputDataset Class +# ===================================================== + + @register_input_dataset(InstrumentType.UNDERWATER_ST) class Underwater_STInputDataset(InputDataset): """Input dataset for Underwater_ST instrument.""" @@ -74,6 +91,11 @@ def get_datasets_dict(self) -> dict: } +# ===================================================== +# SECTION: Instrument Class +# ===================================================== + + @register_instrument(InstrumentType.UNDERWATER_ST) class Underwater_STInstrument(Instrument): """Underwater_ST instrument class.""" diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 641015f5..98378c98 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -10,6 +10,10 @@ from virtualship.models.spacetime import Spacetime from virtualship.utils import register_input_dataset, register_instrument +# ===================================================== +# SECTION: Dataclass +# ===================================================== + @dataclass class XBT: @@ -23,6 +27,10 @@ class XBT: deceleration_coefficient: float +# ===================================================== +# SECTION: Particle Class +# ===================================================== + _XBTParticle = JITParticle.add_variables( [ Variable("temperature", dtype=np.float32, initial=np.nan), @@ -33,6 +41,10 @@ class XBT: ] ) +# ===================================================== +# SECTION: Kernels +# ===================================================== + def _sample_temperature(particle, fieldset, time): particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] @@ -56,6 +68,11 @@ def _xbt_cast(particle, fieldset, time): particle_ddepth = particle.max_depth - particle.depth +# ===================================================== +# SECTION: InputDataset Class +# ===================================================== + + @register_input_dataset(InstrumentType.XBT) class XBTInputDataset(InputDataset): """Input dataset for XBT instrument.""" @@ -101,6 +118,11 @@ def get_datasets_dict(self) -> dict: } +# ===================================================== +# SECTION: Instrument Class +# ===================================================== + + @register_instrument(InstrumentType.XBT) class XBTInstrument(Instrument): """XBT instrument class.""" diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index c70de855..54021176 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -341,7 +341,6 @@ class XBTConfig(pydantic.BaseModel): class InstrumentsConfig(pydantic.BaseModel): - # TODO: refactor potential for this? Move explicit instrument_config's away from models/ dir? """Configuration of instruments.""" argo_float_config: ArgoFloatConfig | None = None From 37530329840f87ad08e1e7c0bd067a7fdcbc9515 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 3 Nov 2025 12:50:10 +0000 Subject: [PATCH 41/97] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/instruments/base.py | 2 +- src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 2 +- src/virtualship/instruments/ship_underwater_st.py | 2 +- src/virtualship/instruments/xbt.py | 2 +- src/virtualship/models/expedition.py | 2 +- tests/instruments/test_ctd.py | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index b2ebe988..857e9655 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 573836b4..64961605 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,7 +4,6 @@ from typing import ClassVar import numpy as np - from parcels import ( AdvectionRK4, JITParticle, @@ -12,6 +11,7 @@ StatusCode, Variable, ) + from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 3c6a333b..93ff8ed8 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -4,9 +4,9 @@ import copernicusmarine import numpy as np +from parcels import Field, FieldSet from yaspin import yaspin -from parcels import Field, FieldSet from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash from virtualship.errors import CopernicusCatalogueError from virtualship.models import Expedition, SpaceTimeRegion diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 5080c006..2c6dc56e 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models import Spacetime diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 28dd55c3..23c978f7 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 0ce78624..f854d51f 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable + from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 946e1a9b..161bb184 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 98378c98..68502533 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 54021176..d6727f69 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -9,8 +9,8 @@ import pydantic import pyproj import yaml - from parcels import Field + from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.instruments.types import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta diff --git a/tests/instruments/test_ctd.py b/tests/instruments/test_ctd.py index 0a8edcfa..14e0a276 100644 --- a/tests/instruments/test_ctd.py +++ b/tests/instruments/test_ctd.py @@ -9,8 +9,8 @@ import numpy as np import xarray as xr - from parcels import Field, FieldSet + from virtualship.instruments.ctd import CTD, simulate_ctd from virtualship.models import Location, Spacetime From b7904e8995cdeacf53538623e5efdf51d7268f8e Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 14:38:10 +0100 Subject: [PATCH 42/97] Refactor type hints and improve test coverage for instruments and utils --- src/virtualship/instruments/base.py | 41 +++++++------ src/virtualship/instruments/ctd.py | 8 ++- tests/cli/test_fetch.py | 27 --------- tests/instruments/test_base.py | 92 +++++++++++++++++++++-------- tests/test_utils.py | 18 +++++- 5 files changed, 113 insertions(+), 73 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 3c6a333b..0bb77c33 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,6 +1,7 @@ import abc from datetime import timedelta from pathlib import Path +from typing import TYPE_CHECKING import copernicusmarine import numpy as np @@ -9,9 +10,11 @@ from parcels import Field, FieldSet from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash from virtualship.errors import CopernicusCatalogueError -from virtualship.models import Expedition, SpaceTimeRegion from virtualship.utils import ship_spinner +if TYPE_CHECKING: + from virtualship.models import Expedition, SpaceTimeRegion + PRODUCT_IDS = { "phys": { "reanalysis": "cmems_mod_glo_phy_my_0.083deg_P1D-m", @@ -60,7 +63,7 @@ def __init__( max_depth: float, data_dir: str, credentials: dict, - space_time_region: SpaceTimeRegion, + space_time_region: "SpaceTimeRegion", ): """Initialise input dataset.""" self.name = name @@ -185,22 +188,7 @@ def _select_product_id( "No suitable product found in the Copernicus Marine Catalogue for the scheduled time and variable." ) - def start_end_in_product_timerange( - selected_id, schedule_start, schedule_end, username, password - ): - ds_selected = copernicusmarine.open_dataset( - selected_id, username=username, password=password - ) - time_values = ds_selected["time"].values - import numpy as np - - time_min, time_max = np.min(time_values), np.max(time_values) - return ( - np.datetime64(schedule_start) >= time_min - and np.datetime64(schedule_end) <= time_max - ) - - if start_end_in_product_timerange( + if self._start_end_in_product_timerange( selected_id, schedule_start, schedule_end, username, password ): return selected_id @@ -211,6 +199,21 @@ def start_end_in_product_timerange( else BGC_ANALYSIS_IDS[variable] ) + def _start_end_in_product_timerange( + self, selected_id, schedule_start, schedule_end, username, password + ): + ds_selected = copernicusmarine.open_dataset( + selected_id, username=username, password=password + ) + time_values = ds_selected["time"].values + import numpy as np + + time_min, time_max = np.min(time_values), np.max(time_values) + return ( + np.datetime64(schedule_start) >= time_min + and np.datetime64(schedule_end) <= time_max + ) + class Instrument(abc.ABC): """Base class for instruments and their simulation.""" @@ -218,7 +221,7 @@ class Instrument(abc.ABC): def __init__( self, name: str, - expedition: Expedition, + expedition: "Expedition", directory: Path | str, filenames: dict, variables: dict, diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 5080c006..8e524f4a 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -1,13 +1,15 @@ from dataclasses import dataclass from datetime import timedelta -from typing import ClassVar +from typing import TYPE_CHECKING, ClassVar import numpy as np from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType -from virtualship.models import Spacetime + +if TYPE_CHECKING: + from virtualship.models.spacetime import Spacetime from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument # TODO: add some kind of check that each instrument has a dataclass, particle class, InputDataset class and Instrument class? @@ -23,7 +25,7 @@ class CTD: """CTD configuration.""" name: ClassVar[str] = "CTD" - spacetime: Spacetime + spacetime: "Spacetime" min_depth: float max_depth: float diff --git a/tests/cli/test_fetch.py b/tests/cli/test_fetch.py index 9725601a..4c041dbe 100644 --- a/tests/cli/test_fetch.py +++ b/tests/cli/test_fetch.py @@ -17,8 +17,6 @@ get_existing_download, hash_model, hash_to_filename, - select_product_id, - start_end_in_product_timerange, ) from virtualship.models import Expedition from virtualship.utils import EXPEDITION, get_example_expedition @@ -98,31 +96,6 @@ def test_complete_download(tmp_path): assert_complete_download(tmp_path) -@pytest.mark.usefixtures("copernicus_no_download") -def test_select_product_id(expedition): - """Should return the physical reanalysis product id via the timings prescribed in the static schedule.yaml file.""" - result = select_product_id( - physical=True, - schedule_start=expedition.schedule.space_time_region.time_range.start_time, - schedule_end=expedition.schedule.space_time_region.time_range.end_time, - username="test", - password="test", - ) - assert result == "cmems_mod_glo_phy_my_0.083deg_P1D-m" - - -@pytest.mark.usefixtures("copernicus_no_download") -def test_start_end_in_product_timerange(expedition): - """Should return True for valid range ass determined by the static schedule.yaml file.""" - assert start_end_in_product_timerange( - selected_id="cmems_mod_glo_phy_my_0.083deg_P1D-m", - schedule_start=expedition.schedule.space_time_region.time_range.start_time, - schedule_end=expedition.schedule.space_time_region.time_range.end_time, - username="test", - password="test", - ) - - def test_assert_complete_download_complete(tmp_path): # Setup DownloadMetadata(download_complete=True).to_yaml(tmp_path / DOWNLOAD_METADATA) diff --git a/tests/instruments/test_base.py b/tests/instruments/test_base.py index f4192092..65e4ae0e 100644 --- a/tests/instruments/test_base.py +++ b/tests/instruments/test_base.py @@ -1,9 +1,12 @@ import datetime -from unittest.mock import patch +from pathlib import Path +from unittest.mock import MagicMock, patch +import numpy as np import pytest +import xarray as xr -from virtualship.instruments.base import InputDataset +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.space_time_region import ( SpaceTimeRegion, @@ -12,6 +15,18 @@ ) from virtualship.utils import get_input_dataset_class +# test dataclass, particle class, kernels, etc. are defined for each instrument + + +# TODO: add all the other things here like particle class, kernels, etc. +def test_all_instruments_have_input_class(): + for instrument in InstrumentType: + input_class = get_input_dataset_class(instrument) + assert input_class is not None, f"No input_class for {instrument}" + + +# test InputDataset class + class DummyInputDataset(InputDataset): """A minimal InputDataset subclass for testing purposes.""" @@ -20,7 +35,7 @@ def get_datasets_dict(self): """Return a dummy datasets dict for testing.""" return { "dummy": { - "dataset_id": "test_id", + "physical": True, "variables": ["var1"], "output_filename": "dummy.nc", } @@ -48,21 +63,6 @@ def dummy_space_time_region(): ) -def test_inputdataset_abstract_instantiation(): - # instantiation should not be allowed - with pytest.raises(TypeError): - InputDataset( - name="test", - latlon_buffer=0, - datetime_buffer=0, - min_depth=0, - max_depth=10, - data_dir=".", - credentials={"username": "u", "password": "p"}, - space_time_region=None, - ) - - def test_dummyinputdataset_initialization(dummy_space_time_region): ds = DummyInputDataset( name="test", @@ -83,8 +83,23 @@ def test_dummyinputdataset_initialization(dummy_space_time_region): assert ds.credentials["username"] == "u" +@patch("virtualship.instruments.base.copernicusmarine.open_dataset") @patch("virtualship.instruments.base.copernicusmarine.subset") -def test_download_data_calls_subset(mock_subset, dummy_space_time_region): +def test_download_data_calls_subset( + mock_subset, mock_open_dataset, dummy_space_time_region +): + """Test that download_data calls the subset function correctly, will also test Copernicus Marine product id search logic.""" + mock_open_dataset.return_value = xr.Dataset( + { + "time": ( + "time", + [ + np.datetime64("1993-01-01T00:00:00"), + np.datetime64("2023-01-01T01:00:00"), + ], + ) + } + ) ds = DummyInputDataset( name="test", latlon_buffer=0.5, @@ -99,7 +114,38 @@ def test_download_data_calls_subset(mock_subset, dummy_space_time_region): assert mock_subset.called -def test_all_instruments_have_input_class(): - for instrument in InstrumentType: - input_class = get_input_dataset_class(instrument) - assert input_class is not None, f"No input_class for {instrument}" +# test Instrument class + + +class DummyInstrument(Instrument): + """Minimal concrete Instrument for testing.""" + + def simulate(self, data_dir, measurements, out_path): + """Dummy simulate implementation for test.""" + self.simulate_called = True + + +@patch("virtualship.instruments.base.FieldSet") +@patch("virtualship.instruments.base.get_existing_download") +@patch("virtualship.instruments.base.get_space_time_region_hash") +def test_load_input_data_calls(mock_hash, mock_get_download, mock_FieldSet): + """Test Instrument.load_input_data with mocks.""" + mock_hash.return_value = "hash" + mock_get_download.return_value = Path("/tmp/data") + mock_fieldset = MagicMock() + mock_FieldSet.from_netcdf.return_value = mock_fieldset + mock_fieldset.gridset.grids = [MagicMock(negate_depth=MagicMock())] + mock_fieldset.__getitem__.side_effect = lambda k: MagicMock() + dummy = DummyInstrument( + name="test", + expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), + directory="/tmp", + filenames={"A": "a.nc"}, + variables={"A": "a"}, + add_bathymetry=False, + allow_time_extrapolation=False, + verbose_progress=False, + ) + fieldset = dummy.load_input_data() + assert mock_FieldSet.from_netcdf.called + assert fieldset == mock_fieldset diff --git a/tests/test_utils.py b/tests/test_utils.py index 0dcebd79..bb8208f6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,4 @@ -from virtualship.models import Expedition +from virtualship.models.expedition import Expedition from virtualship.utils import get_example_expedition @@ -12,3 +12,19 @@ def test_valid_example_expedition(tmp_path): file.write(get_example_expedition()) Expedition.from_yaml(path) + + +def test_instrument_registry_updates(): + from virtualship import utils + + class DummyInputDataset: + pass + + class DummyInstrument: + pass + + utils.register_input_dataset("DUMMY_TYPE")(DummyInputDataset) + utils.register_instrument("DUMMY_TYPE")(DummyInstrument) + + assert utils.INPUT_DATASET_MAP["DUMMY_TYPE"] is DummyInputDataset + assert utils.INSTRUMENT_CLASS_MAP["DUMMY_TYPE"] is DummyInstrument From 7d1c575c7b8bc309db7e6e147357ab83a6c19f7a Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 15:05:31 +0100 Subject: [PATCH 43/97] Remove TODO comments and tidy up imports in test files --- src/virtualship/instruments/ctd.py | 3 --- src/virtualship/models/expedition.py | 1 - tests/instruments/test_base.py | 1 - 3 files changed, 5 deletions(-) diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 8e524f4a..0164ef94 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -12,9 +12,6 @@ from virtualship.models.spacetime import Spacetime from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument -# TODO: add some kind of check that each instrument has a dataclass, particle class, InputDataset class and Instrument class? -# TODO: probably as a test - # ===================================================== # SECTION: Dataclass # ===================================================== diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 54021176..dc198f51 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -131,7 +131,6 @@ def verify( ) # check if all waypoints are in water using bathymetry data - # TODO: tests should be updated to check this! # TODO: write test that checks that will flag when waypoint is on land!! [add to existing suite of fail .verify() tests in test_expedition.py] # TODO: need to do an overhaul of the DATA which is in tests/expedition/expedition_dir - don't think it's currently suitable! land_waypoints = [] diff --git a/tests/instruments/test_base.py b/tests/instruments/test_base.py index 65e4ae0e..0a58efcd 100644 --- a/tests/instruments/test_base.py +++ b/tests/instruments/test_base.py @@ -18,7 +18,6 @@ # test dataclass, particle class, kernels, etc. are defined for each instrument -# TODO: add all the other things here like particle class, kernels, etc. def test_all_instruments_have_input_class(): for instrument in InstrumentType: input_class = get_input_dataset_class(instrument) From e74a5ada099226c60704b991af42397e1e5bd634 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 15:26:15 +0100 Subject: [PATCH 44/97] Refactor bathymetry error handling, update verification methods and remove unused function --- src/virtualship/models/expedition.py | 40 ++++++++-------------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 7c109c99..af85c1c5 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -5,12 +5,11 @@ from pathlib import Path from typing import TYPE_CHECKING -import numpy as np import pydantic import pyproj import yaml -from parcels import Field +from parcels import Field from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.instruments.types import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta @@ -19,7 +18,7 @@ from .space_time_region import SpaceTimeRegion if TYPE_CHECKING: - from parcels import FieldSet + pass projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") @@ -132,7 +131,6 @@ def verify( # check if all waypoints are in water using bathymetry data # TODO: write test that checks that will flag when waypoint is on land!! [add to existing suite of fail .verify() tests in test_expedition.py] - # TODO: need to do an overhaul of the DATA which is in tests/expedition/expedition_dir - don't think it's currently suitable! land_waypoints = [] if data_dir is not None: bathymetry_path = data_dir.joinpath("bathymetry.nc") @@ -147,18 +145,19 @@ def verify( f"Problem loading bathymetry data (used to verify waypoints are in water): {e}" ) from e for wp_i, wp in enumerate(self.waypoints): - bathy = bathymetry_field.eval( - 0, # time - 0, # depth (surface) - wp.location.lat, - wp.location.lon, - ) - if np.isnan(bathy) or bathy <= 0: + try: + bathymetry_field.eval( + 0, # time + 0, # depth (surface) + wp.location.lat, + wp.location.lon, + ) + except Exception: land_waypoints.append((wp_i, wp)) if len(land_waypoints) > 0: raise ScheduleError( - f"The following waypoints are on land: {['#' + str(wp_i + 1) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" + f"The following waypoint(s) throw(s) error(s): {['#' + str(wp_i + 1) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}\n\nINFO: They are likely on land (bathymetry data cannot be interpolated to their location(s)).\n" ) elif not ignore_missing_bathymetry: raise ScheduleError( @@ -431,20 +430,3 @@ def verify(self, expedition: Expedition) -> None: raise InstrumentsConfigError( f"Expedition includes instrument '{inst_type.value}', but instruments_config does not provide configuration for it." ) - - -def _is_on_land_zero_uv(fieldset: FieldSet, waypoint: Waypoint) -> bool: - """ - Check if waypoint is on land by assuming zero velocity means land. - - :param fieldset: The fieldset to sample the velocity from. - :param waypoint: The waypoint to check. - :returns: If the waypoint is on land. - """ - return fieldset.UV.eval( - 0, - fieldset.gridset.grids[0].depth[0], - waypoint.location.lat, - waypoint.location.lon, - applyConversion=False, - ) == (0.0, 0.0) From 56d8fd580900e08126ebeb6a9106e5662b09c8d3 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 5 Nov 2025 12:53:49 +0100 Subject: [PATCH 45/97] move product id selection logic to utils --- src/virtualship/instruments/base.py | 128 +--------------------------- src/virtualship/utils.py | 124 +++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 125 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index a35aa81a..0f530702 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -4,49 +4,15 @@ from typing import TYPE_CHECKING import copernicusmarine -import numpy as np -from parcels import Field, FieldSet from yaspin import yaspin +from parcels import Field, FieldSet from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash -from virtualship.errors import CopernicusCatalogueError -from virtualship.utils import ship_spinner +from virtualship.utils import _select_product_id, ship_spinner if TYPE_CHECKING: from virtualship.models import Expedition, SpaceTimeRegion -PRODUCT_IDS = { - "phys": { - "reanalysis": "cmems_mod_glo_phy_my_0.083deg_P1D-m", - "reanalysis_interim": "cmems_mod_glo_phy_myint_0.083deg_P1D-m", - "analysis": "cmems_mod_glo_phy_anfc_0.083deg_P1D-m", - }, - "bgc": { - "reanalysis": "cmems_mod_glo_bgc_my_0.25deg_P1D-m", - "reanalysis_interim": "cmems_mod_glo_bgc_myint_0.25deg_P1D-m", - "analysis": None, # will be set per variable - }, -} - -BGC_ANALYSIS_IDS = { - "o2": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", - "chl": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", - "no3": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", - "po4": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", - "ph": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", - "phyc": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", - "nppv": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", -} - -MONTHLY_BGC_REANALYSIS_IDS = { - "ph": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", - "phyc": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", -} -MONTHLY_BGC_REANALYSIS_INTERIM_IDS = { - "ph": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", - "phyc": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", -} - class InputDataset(abc.ABC): """Base class for instrument input datasets.""" @@ -111,7 +77,7 @@ def download_data(self) -> None: else: variable = dataset.get("variables")[0] # BGC variables, special case - dataset_id = self._select_product_id( + dataset_id = _select_product_id( physical=physical, schedule_start=self.space_time_region.time_range.start_time, schedule_end=self.space_time_region.time_range.end_time, @@ -126,94 +92,6 @@ def download_data(self) -> None: } copernicusmarine.subset(**download_args) - def _select_product_id( - self, - physical: bool, - schedule_start, - schedule_end, - username: str, - password: str, - variable: str | None = None, - ) -> str: - """Determine which copernicus product id should be selected (reanalysis, reanalysis-interim, analysis & forecast), for prescribed schedule and physical vs. BGC.""" - key = "phys" if physical else "bgc" - selected_id = None - - for period, pid in PRODUCT_IDS[key].items(): - # for BGC analysis, set pid per variable - if key == "bgc" and period == "analysis": - if variable is None or variable not in BGC_ANALYSIS_IDS: - continue - pid = BGC_ANALYSIS_IDS[variable] - # for BGC reanalysis, check if requires monthly product - if ( - key == "bgc" - and period == "reanalysis" - and variable in MONTHLY_BGC_REANALYSIS_IDS - ): - monthly_pid = MONTHLY_BGC_REANALYSIS_IDS[variable] - ds_monthly = copernicusmarine.open_dataset( - monthly_pid, - username=username, - password=password, - ) - time_end_monthly = ds_monthly["time"][-1].values - if np.datetime64(schedule_end) <= time_end_monthly: - pid = monthly_pid - # for BGC reanalysis_interim, check if requires monthly product - if ( - key == "bgc" - and period == "reanalysis_interim" - and variable in MONTHLY_BGC_REANALYSIS_INTERIM_IDS - ): - monthly_pid = MONTHLY_BGC_REANALYSIS_INTERIM_IDS[variable] - ds_monthly = copernicusmarine.open_dataset( - monthly_pid, username=username, password=password - ) - time_end_monthly = ds_monthly["time"][-1].values - if np.datetime64(schedule_end) <= time_end_monthly: - pid = monthly_pid - if pid is None: - continue - ds = copernicusmarine.open_dataset( - pid, username=username, password=password - ) - time_end = ds["time"][-1].values - if np.datetime64(schedule_end) <= time_end: - selected_id = pid - break - - if selected_id is None: - raise CopernicusCatalogueError( - "No suitable product found in the Copernicus Marine Catalogue for the scheduled time and variable." - ) - - if self._start_end_in_product_timerange( - selected_id, schedule_start, schedule_end, username, password - ): - return selected_id - else: - return ( - PRODUCT_IDS["phys"]["analysis"] - if physical - else BGC_ANALYSIS_IDS[variable] - ) - - def _start_end_in_product_timerange( - self, selected_id, schedule_start, schedule_end, username, password - ): - ds_selected = copernicusmarine.open_dataset( - selected_id, username=username, password=password - ) - time_values = ds_selected["time"].values - import numpy as np - - time_min, time_max = np.min(time_values), np.max(time_values) - return ( - np.datetime64(schedule_start) >= time_min - and np.datetime64(schedule_end) <= time_max - ) - class Instrument(abc.ABC): """Base class for instruments and their simulation.""" diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 324f8c67..8f1fadd4 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -8,7 +8,11 @@ from pathlib import Path from typing import TYPE_CHECKING, TextIO +import copernicusmarine +import numpy as np + from parcels import FieldSet +from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: from virtualship.models import Expedition @@ -293,3 +297,123 @@ def add_dummy_UV(fieldset: FieldSet): raise ValueError( "Cannot determine time_origin for dummy UV fields. Assert T or o2 exists in fieldset." ) from None + + +# Copernicus Marine product IDs + +PRODUCT_IDS = { + "phys": { + "reanalysis": "cmems_mod_glo_phy_my_0.083deg_P1D-m", + "reanalysis_interim": "cmems_mod_glo_phy_myint_0.083deg_P1D-m", + "analysis": "cmems_mod_glo_phy_anfc_0.083deg_P1D-m", + }, + "bgc": { + "reanalysis": "cmems_mod_glo_bgc_my_0.25deg_P1D-m", + "reanalysis_interim": "cmems_mod_glo_bgc_myint_0.25deg_P1D-m", + "analysis": None, # will be set per variable + }, +} + +BGC_ANALYSIS_IDS = { + "o2": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "chl": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "no3": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "po4": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "ph": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", + "phyc": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "nppv": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", +} + +MONTHLY_BGC_REANALYSIS_IDS = { + "ph": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", + "phyc": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", +} +MONTHLY_BGC_REANALYSIS_INTERIM_IDS = { + "ph": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", + "phyc": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", +} + + +def _select_product_id( + physical: bool, + schedule_start, + schedule_end, + username: str, + password: str, + variable: str | None = None, +) -> str: + """Determine which copernicus product id should be selected (reanalysis, reanalysis-interim, analysis & forecast), for prescribed schedule and physical vs. BGC.""" + key = "phys" if physical else "bgc" + selected_id = None + + for period, pid in PRODUCT_IDS[key].items(): + # for BGC analysis, set pid per variable + if key == "bgc" and period == "analysis": + if variable is None or variable not in BGC_ANALYSIS_IDS: + continue + pid = BGC_ANALYSIS_IDS[variable] + # for BGC reanalysis, check if requires monthly product + if ( + key == "bgc" + and period == "reanalysis" + and variable in MONTHLY_BGC_REANALYSIS_IDS + ): + monthly_pid = MONTHLY_BGC_REANALYSIS_IDS[variable] + ds_monthly = copernicusmarine.open_dataset( + monthly_pid, + username=username, + password=password, + ) + time_end_monthly = ds_monthly["time"][-1].values + if np.datetime64(schedule_end) <= time_end_monthly: + pid = monthly_pid + # for BGC reanalysis_interim, check if requires monthly product + if ( + key == "bgc" + and period == "reanalysis_interim" + and variable in MONTHLY_BGC_REANALYSIS_INTERIM_IDS + ): + monthly_pid = MONTHLY_BGC_REANALYSIS_INTERIM_IDS[variable] + ds_monthly = copernicusmarine.open_dataset( + monthly_pid, username=username, password=password + ) + time_end_monthly = ds_monthly["time"][-1].values + if np.datetime64(schedule_end) <= time_end_monthly: + pid = monthly_pid + if pid is None: + continue + ds = copernicusmarine.open_dataset(pid, username=username, password=password) + time_end = ds["time"][-1].values + if np.datetime64(schedule_end) <= time_end: + selected_id = pid + break + + if selected_id is None: + raise CopernicusCatalogueError( + "No suitable product found in the Copernicus Marine Catalogue for the scheduled time and variable." + ) + + if _start_end_in_product_timerange( + selected_id, schedule_start, schedule_end, username, password + ): + return selected_id + else: + return ( + PRODUCT_IDS["phys"]["analysis"] if physical else BGC_ANALYSIS_IDS[variable] + ) + + +def _start_end_in_product_timerange( + selected_id, schedule_start, schedule_end, username, password +): + ds_selected = copernicusmarine.open_dataset( + selected_id, username=username, password=password + ) + time_values = ds_selected["time"].values + import numpy as np + + time_min, time_max = np.min(time_values), np.max(time_values) + return ( + np.datetime64(schedule_start) >= time_min + and np.datetime64(schedule_end) <= time_max + ) From c447dd86c446e7a77e2aba2a87d6a6efdff9e47b Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 5 Nov 2025 14:11:46 +0100 Subject: [PATCH 46/97] update --- src/virtualship/instruments/base.py | 69 +++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 0f530702..0e5ff37a 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -96,6 +96,11 @@ def download_data(self) -> None: class Instrument(abc.ABC): """Base class for instruments and their simulation.""" + #! TODO List: + # TODO: update documentation/quickstart + # TODO: update tests + # TODO: if use direct ingestion as primary data sourcing, can substantially cut code base (including _fetch.py, InputDataset objects). Consider this for Parcels v4 transition. + def __init__( self, name: str, @@ -107,6 +112,7 @@ def __init__( allow_time_extrapolation: bool, verbose_progress: bool, bathymetry_file: str = "bathymetry.nc", + direct: bool = False, ): """Initialise instrument.""" self.name = name @@ -124,27 +130,54 @@ def __init__( self.add_bathymetry = add_bathymetry self.allow_time_extrapolation = allow_time_extrapolation self.verbose_progress = verbose_progress + self.direct = direct def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" - # TODO: tests need updating...! - - try: - data_dir = self._get_data_dir(self.directory) - joined_filepaths = { - key: data_dir.joinpath(filename) - for key, filename in self.filenames.items() - } - fieldset = FieldSet.from_netcdf( - joined_filepaths, - self.variables, - self.dimensions, - allow_time_extrapolation=self.allow_time_extrapolation, - ) - except FileNotFoundError as e: - raise FileNotFoundError( - f"Input data for instrument {self.name} not found. Have you run the `virtualship fetch` command??" - ) from e + if self.direct: # if direct ingestion from Copernicus Marine is enabled + try: + # ds = copernicusmarine.open_dataset( + # dataset_id="PHYS_REANALYSIS_ID", + # dataset_part="default", + # minimum_longitude=self.expedition.schedule.space_time_region.spatial_range.minimum_longitude, + # maximum_longitude=self.expedition.schedule.space_time_region.spatial_range.maximum_longitude, + # minimum_latitude=self.expedition.schedule.space_time_region.spatial_range.minimum_latitude, + # maximum_latitude=self.expedition.schedule.space_time_region.spatial_range.maximum_latitude, + # variables=["uo", "vo", "so", "thetao"], + # start_datetime=self.expedition.schedule.space_time_region.time_range.start_time, + # end_datetime=self.expedition.schedule.space_time_region.time_range.end_time, + # coordinates_selection_method="outside", + # ) + + #! TODO: FIX! + fieldset = copernicusmarine.FieldSet.from_copernicus( + self.expedition.schedule.space_time_region, + self.variables, + self.dimensions, + allow_time_extrapolation=self.allow_time_extrapolation, + ) + except FileNotFoundError as e: + raise FileNotFoundError( + "ERROR" # TODO: improve error message! + ) from e + + else: # from fetched data on disk + try: + data_dir = self._get_data_dir(self.directory) + joined_filepaths = { + key: data_dir.joinpath(filename) + for key, filename in self.filenames.items() + } + fieldset = FieldSet.from_netcdf( + joined_filepaths, + self.variables, + self.dimensions, + allow_time_extrapolation=self.allow_time_extrapolation, + ) + except FileNotFoundError as e: + raise FileNotFoundError( + f"Input data for instrument {self.name} not found. Have you run the `virtualship fetch` command??" + ) from e # interpolation methods for var in (v for v in self.variables if v not in ("U", "V")): From 870271bc04626a61c0568efae53874d29a0e4bd1 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 5 Nov 2025 16:53:31 +0100 Subject: [PATCH 47/97] first draft direct ingestion via copernicusmarine (CTD only) --- src/virtualship/cli/commands.py | 11 +++- src/virtualship/expedition/do_expedition.py | 25 +++++--- src/virtualship/instruments/base.py | 70 +++++++++++++-------- src/virtualship/instruments/ctd.py | 5 +- src/virtualship/models/expedition.py | 44 +++++++------ src/virtualship/utils.py | 24 ++++++- 6 files changed, 123 insertions(+), 56 deletions(-) diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index 3e83be3b..0f3c058a 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -106,11 +106,18 @@ def fetch(path: str | Path, username: str | None, password: str | None) -> None: _fetch(path, username, password) +# TODO: also add option to 'stream' via link to dir elsewhere, e.g. simlink or path to data stored elsewhere that isn't expedition dir! @click.command() @click.argument( "path", type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True), ) -def run(path): +@click.option( + "--from-copernicusmarine", + is_flag=True, + default=False, + help="Ingest fieldsets directly via copernicusmarine toolbox.", +) +def run(path, from_copernicusmarine: bool): """Run the expedition.""" - do_expedition(Path(path)) + do_expedition(Path(path), from_copernicusmarine) diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index b857b72c..eaef2b34 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -31,12 +31,16 @@ external_logger = logging.getLogger("parcels.tools.loggers") external_logger.setLevel(logging.WARNING) +# copernicusmarine logger (suppress INFO messages to prevent log being flooded) +logging.getLogger("copernicusmarine").setLevel("ERROR") -def do_expedition(expedition_dir: str | Path) -> None: + +def do_expedition(expedition_dir: str | Path, from_copernicusmarine: bool) -> None: """ Perform an expedition, providing terminal feedback and file output. :param expedition_dir: The base directory for the expedition. + :param from_copernicusmarine: Whether to use direct data ingestion from Copernicus Marine. Should be determined by CLI flag. """ print("\n╔═════════════════════════════════════════════════╗") print("║ VIRTUALSHIP EXPEDITION STATUS ║") @@ -61,12 +65,15 @@ def do_expedition(expedition_dir: str | Path) -> None: print("\n---- WAYPOINT VERIFICATION ----") # verify schedule is valid - data_dir = get_existing_download( - expedition_dir, - get_space_time_region_hash(expedition.schedule.space_time_region), - ) + if from_copernicusmarine: + bathy_data_dir = None + else: + bathy_data_dir = get_existing_download( + expedition_dir, + get_space_time_region_hash(expedition.schedule.space_time_region), + ) - expedition.schedule.verify(expedition.ship_config.ship_speed_knots, data_dir) + expedition.schedule.verify(expedition.ship_config.ship_speed_knots, bathy_data_dir) # simulate the schedule schedule_results = simulate_schedule( @@ -117,7 +124,11 @@ def do_expedition(expedition_dir: str | Path) -> None: measurements = getattr(schedule_results.measurements_to_simulate, attr) # initialise instrument - instrument = instrument_class(expedition=expedition, directory=expedition_dir) + instrument = instrument_class( + expedition=expedition, + directory=expedition_dir, + from_copernicusmarine=from_copernicusmarine, + ) # run simulation instrument.run( diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 0e5ff37a..920b8dbf 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING import copernicusmarine +import xarray as xr from yaspin import yaspin from parcels import Field, FieldSet @@ -112,7 +113,7 @@ def __init__( allow_time_extrapolation: bool, verbose_progress: bool, bathymetry_file: str = "bathymetry.nc", - direct: bool = False, + from_copernicusmarine: bool = False, ): """Initialise instrument.""" self.name = name @@ -130,35 +131,31 @@ def __init__( self.add_bathymetry = add_bathymetry self.allow_time_extrapolation = allow_time_extrapolation self.verbose_progress = verbose_progress - self.direct = direct + self.from_copernicusmarine = from_copernicusmarine def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" - if self.direct: # if direct ingestion from Copernicus Marine is enabled + if self.from_copernicusmarine: try: - # ds = copernicusmarine.open_dataset( - # dataset_id="PHYS_REANALYSIS_ID", - # dataset_part="default", - # minimum_longitude=self.expedition.schedule.space_time_region.spatial_range.minimum_longitude, - # maximum_longitude=self.expedition.schedule.space_time_region.spatial_range.maximum_longitude, - # minimum_latitude=self.expedition.schedule.space_time_region.spatial_range.minimum_latitude, - # maximum_latitude=self.expedition.schedule.space_time_region.spatial_range.maximum_latitude, - # variables=["uo", "vo", "so", "thetao"], - # start_datetime=self.expedition.schedule.space_time_region.time_range.start_time, - # end_datetime=self.expedition.schedule.space_time_region.time_range.end_time, - # coordinates_selection_method="outside", - # ) - - #! TODO: FIX! - fieldset = copernicusmarine.FieldSet.from_copernicus( - self.expedition.schedule.space_time_region, - self.variables, - self.dimensions, - allow_time_extrapolation=self.allow_time_extrapolation, + datasets = [] + for var in self.variables.values(): + physical = ( + True if var in ("uo", "vo", "so", "thetao") else False + ) # TODO: add more if start using new physical variables! Or more dynamic way of determining? + ds = self._get_copernicus_ds( + physical=physical, var=var + ) # user should be prompted for credentials + datasets.append(ds) + + ds_concat = xr.merge(datasets) + fieldset = FieldSet.from_xarray_dataset( + ds_concat, self.variables, self.dimensions, mesh="spherical" ) - except FileNotFoundError as e: + + except Exception as e: raise FileNotFoundError( - "ERROR" # TODO: improve error message! + f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'. " + f"Please check your credentials, network connection, and variable names. Original error: {e}" ) from e else: # from fetched data on disk @@ -176,7 +173,8 @@ def load_input_data(self) -> FieldSet: ) except FileNotFoundError as e: raise FileNotFoundError( - f"Input data for instrument {self.name} not found. Have you run the `virtualship fetch` command??" + f"Input data for instrument {self.name} not found locally. Have you run the `virtualship fetch` command?" + "Alternatively, you can use the `--from-copernicusmarine` option to ingest data directly from Copernicus Marine." ) from e # interpolation methods @@ -230,3 +228,25 @@ def _get_data_dir(self, expedition_dir: Path) -> Path: ) return data_dir + + def _get_copernicus_ds(self, physical: bool, var: str) -> xr.Dataset: + """Get Copernicus Marine dataset for direct ingestion.""" + product_id = _select_product_id( + physical=physical, + schedule_start=self.expedition.schedule.space_time_region.time_range.start_time, + schedule_end=self.expedition.schedule.space_time_region.time_range.end_time, + variable=var if not physical else None, + ) + + return copernicusmarine.open_dataset( + dataset_id=product_id, + dataset_part="default", + minimum_longitude=self.expedition.schedule.space_time_region.spatial_range.minimum_longitude, + maximum_longitude=self.expedition.schedule.space_time_region.spatial_range.maximum_longitude, + minimum_latitude=self.expedition.schedule.space_time_region.spatial_range.minimum_latitude, + maximum_latitude=self.expedition.schedule.space_time_region.spatial_range.maximum_latitude, + variables=["uo", "vo", "so", "thetao"], + start_datetime=self.expedition.schedule.space_time_region.time_range.start_time, + end_datetime=self.expedition.schedule.space_time_region.time_range.end_time, + coordinates_selection_method="outside", + ) diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 8ddb2822..7434da2d 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import TYPE_CHECKING, ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType @@ -122,7 +122,7 @@ def get_datasets_dict(self) -> dict: class CTDInstrument(Instrument): """CTD instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_copernicusmarine): """Initialize CTDInstrument.""" filenames = { "S": f"{CTD.name}_s.nc", @@ -139,6 +139,7 @@ def __init__(self, expedition, directory): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, + from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index af85c1c5..14d9537d 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -12,7 +12,7 @@ from parcels import Field from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.instruments.types import InstrumentType -from virtualship.utils import _validate_numeric_mins_to_timedelta +from virtualship.utils import _get_bathy_data, _validate_numeric_mins_to_timedelta from .location import Location from .space_time_region import SpaceTimeRegion @@ -89,7 +89,7 @@ class Schedule(pydantic.BaseModel): def verify( self, ship_speed: float, - data_dir: str | Path | None, + bathy_data_dir: str | Path | None, ignore_missing_bathymetry: bool = False, *, check_space_time_region: bool = False, @@ -132,18 +132,30 @@ def verify( # check if all waypoints are in water using bathymetry data # TODO: write test that checks that will flag when waypoint is on land!! [add to existing suite of fail .verify() tests in test_expedition.py] land_waypoints = [] - if data_dir is not None: - bathymetry_path = data_dir.joinpath("bathymetry.nc") - try: - bathymetry_field = Field.from_netcdf( - bathymetry_path, - variable=("bathymetry", "deptho"), - dimensions={"lon": "longitude", "lat": "latitude"}, - ) - except Exception as e: - raise ScheduleError( - f"Problem loading bathymetry data (used to verify waypoints are in water): {e}" - ) from e + if not ignore_missing_bathymetry: + if bathy_data_dir is None: + try: + bathymetry_field = _get_bathy_data( + self.space_time_region + ).bathymetry # via copernicusmarine + except Exception as e: + raise ScheduleError( + f"Problem loading bathymetry data (used to verify waypoints are in water) directly via copernicusmarine. \n\n original message: {e}" + ) from e + + else: + bathymetry_path = bathy_data_dir.joinpath("bathymetry.nc") + try: + bathymetry_field = Field.from_netcdf( + bathymetry_path, + variable=("bathymetry", "deptho"), + dimensions={"lon": "longitude", "lat": "latitude"}, + ) + except Exception as e: + raise ScheduleError( + f"Problem loading local bathymetry data (used to verify waypoints are in water). Have you run `virtualship fetch` command?. \n\n original message: {e}" + ) from e + for wp_i, wp in enumerate(self.waypoints): try: bathymetry_field.eval( @@ -159,10 +171,6 @@ def verify( raise ScheduleError( f"The following waypoint(s) throw(s) error(s): {['#' + str(wp_i + 1) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}\n\nINFO: They are likely on land (bathymetry data cannot be interpolated to their location(s)).\n" ) - elif not ignore_missing_bathymetry: - raise ScheduleError( - "Cannot verify waypoints are in water as bathymetry data not found. Have you run `virtualship fetch` command?" - ) # check that ship will arrive on time at each waypoint (in case no unexpected event happen) time = self.waypoints[0].time diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 8f1fadd4..a013d37e 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -338,8 +338,8 @@ def _select_product_id( physical: bool, schedule_start, schedule_end, - username: str, - password: str, + username: str | None = None, + password: str | None = None, variable: str | None = None, ) -> str: """Determine which copernicus product id should be selected (reanalysis, reanalysis-interim, analysis & forecast), for prescribed schedule and physical vs. BGC.""" @@ -417,3 +417,23 @@ def _start_end_in_product_timerange( np.datetime64(schedule_start) >= time_min and np.datetime64(schedule_end) <= time_max ) + + +def _get_bathy_data(space_time_region) -> FieldSet: + """Bathymetry data 'streamed' directly from Copernicus Marine.""" + ds_bathymetry = copernicusmarine.open_dataset( + dataset_id="cmems_mod_glo_phy_my_0.083deg_static", + minimum_longitude=space_time_region.spatial_range.minimum_longitude, + maximum_longitude=space_time_region.spatial_range.maximum_longitude, + minimum_latitude=space_time_region.spatial_range.minimum_latitude, + maximum_latitude=space_time_region.spatial_range.maximum_latitude, + variables=["deptho"], + start_datetime=space_time_region.time_range.start_time, + end_datetime=space_time_region.time_range.end_time, + coordinates_selection_method="outside", + ) + bathymetry_variables = {"bathymetry": "deptho"} + bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} + return FieldSet.from_xarray_dataset( + ds_bathymetry, bathymetry_variables, bathymetry_dimensions + ) From 701e45d5005ee50cc8bad166ae92f1e222d87cca Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 5 Nov 2025 17:23:57 +0100 Subject: [PATCH 48/97] refactor bathymetry data handling and add (temporary) timing for performance evaluation --- src/virtualship/cli/_plan.py | 2 +- src/virtualship/expedition/do_expedition.py | 13 +++++++++++ src/virtualship/instruments/base.py | 24 ++++++++++++++------- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index c27cb741..6aa6ff28 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -1046,7 +1046,7 @@ def save_pressed(self) -> None: # verify schedule expedition_editor.expedition.schedule.verify( ship_speed_value, - data_dir=None, + bathy_data_dir=None, check_space_time_region=True, ignore_missing_bathymetry=True, ) diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index eaef2b34..af9fc4aa 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -42,6 +42,13 @@ def do_expedition(expedition_dir: str | Path, from_copernicusmarine: bool) -> No :param expedition_dir: The base directory for the expedition. :param from_copernicusmarine: Whether to use direct data ingestion from Copernicus Marine. Should be determined by CLI flag. """ + # ################################# TEMPORARY TIMER: START ################################# + import time + + start_time = time.time() + print("[TIMER] Expedition started...") + # ################################# TEMPORARY TIMER: START ################################# + print("\n╔═════════════════════════════════════════════════╗") print("║ VIRTUALSHIP EXPEDITION STATUS ║") print("╚═════════════════════════════════════════════════╝") @@ -145,6 +152,12 @@ def do_expedition(expedition_dir: str | Path, from_copernicusmarine: bool) -> No ) print("\n------------- END -------------\n") + ################################# TEMPORARY TIMER: END ################################# + end_time = time.time() + elapsed = end_time - start_time + print(f"[TIMER] Expedition completed in {elapsed:.2f} seconds.") + ################################# TEMPORARY TIMER: END ################################# + def _load_checkpoint(expedition_dir: Path) -> Checkpoint | None: file_path = expedition_dir.joinpath(CHECKPOINT) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 920b8dbf..ed40e151 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -9,7 +9,7 @@ from parcels import Field, FieldSet from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash -from virtualship.utils import _select_product_id, ship_spinner +from virtualship.utils import _get_bathy_data, _select_product_id, ship_spinner if TYPE_CHECKING: from virtualship.models import Expedition, SpaceTimeRegion @@ -147,7 +147,7 @@ def load_input_data(self) -> FieldSet: ) # user should be prompted for credentials datasets.append(ds) - ds_concat = xr.merge(datasets) + ds_concat = xr.merge(datasets) # TODO: deal with WARNINGS? fieldset = FieldSet.from_xarray_dataset( ds_concat, self.variables, self.dimensions, mesh="spherical" ) @@ -183,16 +183,24 @@ def load_input_data(self) -> FieldSet: # depth negative for g in fieldset.gridset.grids: g.negate_depth() + # bathymetry data if self.add_bathymetry: - bathymetry_field = Field.from_netcdf( - data_dir.joinpath(self.bathymetry_file), - variable=("bathymetry", "deptho"), - dimensions={"lon": "longitude", "lat": "latitude"}, - ) + if self.from_copernicusmarine: + bathymetry_field = _get_bathy_data( + self.expedition.schedule.space_time_region + ).bathymetry + else: + bathymetry_field = Field.from_netcdf( + data_dir.joinpath(self.bathymetry_file), + variable=("bathymetry", "deptho"), + dimensions={"lon": "longitude", "lat": "latitude"}, + ) bathymetry_field.data = -bathymetry_field.data fieldset.add_field(bathymetry_field) - fieldset.computeTimeChunk(0, 1) # read in data already + + # TODO: is this line necessary?! + # fieldset.computeTimeChunk(0, 1) # read in data already return fieldset From 81941155258ca4b068bcf45eaebabf9096366a52 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 6 Nov 2025 08:58:28 +0100 Subject: [PATCH 49/97] update instrument constructors for Copernicus Marine ingestion --- src/virtualship/cli/_fetch.py | 15 ++++++++++++++- src/virtualship/instruments/adcp.py | 5 +++-- src/virtualship/instruments/argo_float.py | 5 +++-- src/virtualship/instruments/base.py | 1 + src/virtualship/instruments/ctd_bgc.py | 5 +++-- src/virtualship/instruments/drifter.py | 5 +++-- src/virtualship/instruments/ship_underwater_st.py | 5 +++-- src/virtualship/instruments/xbt.py | 5 +++-- 8 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index a41687d0..9722fe36 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -39,6 +39,13 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None be provided on prompt, via command line arguments, or via a YAML config file. Run `virtualship fetch` on an expedition for more info. """ + # ################################# TEMPORARY TIMER: START ################################# + import time + + start_time = time.time() + print("[TIMER] Expedition started...") + # ################################# TEMPORARY TIMER: START ################################# + if sum([username is None, password is None]) == 1: raise ValueError("Both username and password must be provided when using CLI.") @@ -51,7 +58,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None expedition.schedule.verify( expedition.ship_config.ship_speed_knots, - data_dir=None, + bathy_data_dir=None, check_space_time_region=True, ignore_missing_bathymetry=True, ) @@ -129,6 +136,12 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None complete_download(download_folder) + ################################# TEMPORARY TIMER: END ################################# + end_time = time.time() + elapsed = end_time - start_time + print(f"[TIMER] Expedition completed in {elapsed:.2f} seconds.") + ################################# TEMPORARY TIMER: END ################################# + def _hash(s: str, *, length: int) -> str: """Create a hash of a string.""" diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 857e9655..2702cbfd 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np -from parcels import ParticleSet, ScipyParticle, Variable +from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( @@ -95,7 +95,7 @@ def get_datasets_dict(self) -> dict: class ADCPInstrument(Instrument): """ADCP instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_copernicusmarine): """Initialize ADCPInstrument.""" filenames = { "U": f"{ADCP.name}_uv.nc", @@ -111,6 +111,7 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, + from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 64961605..30def124 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,6 +4,7 @@ from typing import ClassVar import numpy as np + from parcels import ( AdvectionRK4, JITParticle, @@ -11,7 +12,6 @@ StatusCode, Variable, ) - from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -187,7 +187,7 @@ def get_datasets_dict(self) -> dict: class ArgoFloatInstrument(Instrument): """ArgoFloat instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_copernicusmarine): """Initialize ArgoFloatInstrument.""" filenames = { "U": f"{ArgoFloat.name}_uv.nc", @@ -205,6 +205,7 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, + from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index ed40e151..a0e34e42 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -101,6 +101,7 @@ class Instrument(abc.ABC): # TODO: update documentation/quickstart # TODO: update tests # TODO: if use direct ingestion as primary data sourcing, can substantially cut code base (including _fetch.py, InputDataset objects). Consider this for Parcels v4 transition. + #! TODO: how is this handling credentials?! Seems to work already, are these set up from my previous instances of using copernicusmarine? Therefore users will only have to do it once too? def __init__( self, diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 23c978f7..89173bbf 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -169,7 +169,7 @@ def get_datasets_dict(self) -> dict: class CTD_BGCInstrument(Instrument): """CTD_BGC instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_copernicusmarine): """Initialize CTD_BGCInstrument.""" filenames = { "o2": f"{CTD_BGC.name}_o2.nc", @@ -198,6 +198,7 @@ def __init__(self, expedition, directory): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, + from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index f854d51f..182100c2 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable +from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -108,7 +108,7 @@ def get_datasets_dict(self) -> dict: class DrifterInstrument(Instrument): """Drifter instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_copernicusmarine): """Initialize DrifterInstrument.""" filenames = { "U": f"{Drifter.name}_uv.nc", @@ -125,6 +125,7 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, + from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 161bb184..5f793d3d 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np -from parcels import ParticleSet, ScipyParticle, Variable +from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument @@ -100,7 +100,7 @@ def get_datasets_dict(self) -> dict: class Underwater_STInstrument(Instrument): """Underwater_ST instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_copernicusmarine): """Initialize Underwater_STInstrument.""" filenames = { "S": f"{Underwater_ST.name}_s.nc", @@ -117,6 +117,7 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, + from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 68502533..ab11ed67 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -127,7 +127,7 @@ def get_datasets_dict(self) -> dict: class XBTInstrument(Instrument): """XBT instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_copernicusmarine): """Initialize XBTInstrument.""" filenames = { "U": f"{XBT.name}_uv.nc", @@ -145,6 +145,7 @@ def __init__(self, expedition, directory): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, + from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: From 446b8d0b503a395c99806f0e554501ebf3c80e72 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 6 Nov 2025 09:16:47 +0100 Subject: [PATCH 50/97] move expedition/do_expedition.py to cli/_run.py, rename Instrument.run() to Instrument.execute() --- .../do_expedition.py => cli/_run.py} | 21 +++++++++---------- src/virtualship/cli/commands.py | 4 ++-- src/virtualship/instruments/base.py | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) rename src/virtualship/{expedition/do_expedition.py => cli/_run.py} (94%) diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/cli/_run.py similarity index 94% rename from src/virtualship/expedition/do_expedition.py rename to src/virtualship/cli/_run.py index b857b72c..fb67ba5d 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/cli/_run.py @@ -8,6 +8,13 @@ import pyproj from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash +from virtualship.expedition.checkpoint import Checkpoint +from virtualship.expedition.expedition_cost import expedition_cost +from virtualship.expedition.simulate_schedule import ( + MeasurementsToSimulate, + ScheduleProblem, + simulate_schedule, +) from virtualship.models import Schedule from virtualship.utils import ( CHECKPOINT, @@ -15,14 +22,6 @@ get_instrument_class, ) -from .checkpoint import Checkpoint -from .expedition_cost import expedition_cost -from .simulate_schedule import ( - MeasurementsToSimulate, - ScheduleProblem, - simulate_schedule, -) - # projection used to sail between waypoints projection = pyproj.Geod(ellps="WGS84") @@ -32,7 +31,7 @@ external_logger.setLevel(logging.WARNING) -def do_expedition(expedition_dir: str | Path) -> None: +def _run(expedition_dir: str | Path) -> None: """ Perform an expedition, providing terminal feedback and file output. @@ -119,8 +118,8 @@ def do_expedition(expedition_dir: str | Path) -> None: # initialise instrument instrument = instrument_class(expedition=expedition, directory=expedition_dir) - # run simulation - instrument.run( + # execute simulation + instrument.execute( measurements=measurements, out_path=expedition_dir.joinpath("results", f"{itype.name.lower()}.zarr"), ) diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index 3e83be3b..4571174d 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -5,7 +5,7 @@ from virtualship import utils from virtualship.cli._fetch import _fetch from virtualship.cli._plan import _plan -from virtualship.expedition.do_expedition import do_expedition +from virtualship.cli._run import _run from virtualship.utils import ( EXPEDITION, mfp_to_yaml, @@ -113,4 +113,4 @@ def fetch(path: str | Path, username: str | None, password: str | None) -> None: ) def run(path): """Run the expedition.""" - do_expedition(Path(path)) + _run(Path(path)) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 0f530702..4497764c 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -169,7 +169,7 @@ def load_input_data(self) -> FieldSet: def simulate(self, data_dir: Path, measurements: list, out_path: str | Path): """Simulate instrument measurements.""" - def run(self, measurements: list, out_path: str | Path) -> None: + def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" # TODO: this will have to be able to handle the non-spinner/instead progress bar for drifters and argos! From c97d31937a76c178ae8314c3e5ac2c367a4a3f3f Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 6 Nov 2025 09:40:00 +0100 Subject: [PATCH 51/97] move checkpoint class to models, move expedition_cost() to utils.py --- src/virtualship/cli/_run.py | 2 +- src/virtualship/expedition/expedition_cost.py | 27 ------------------- .../{expedition => models}/checkpoint.py | 0 src/virtualship/utils.py | 24 +++++++++++++++++ 4 files changed, 25 insertions(+), 28 deletions(-) delete mode 100644 src/virtualship/expedition/expedition_cost.py rename src/virtualship/{expedition => models}/checkpoint.py (100%) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index fb67ba5d..46815166 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -8,7 +8,6 @@ import pyproj from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash -from virtualship.expedition.checkpoint import Checkpoint from virtualship.expedition.expedition_cost import expedition_cost from virtualship.expedition.simulate_schedule import ( MeasurementsToSimulate, @@ -16,6 +15,7 @@ simulate_schedule, ) from virtualship.models import Schedule +from virtualship.models.checkpoint import Checkpoint from virtualship.utils import ( CHECKPOINT, _get_expedition, diff --git a/src/virtualship/expedition/expedition_cost.py b/src/virtualship/expedition/expedition_cost.py deleted file mode 100644 index cab6ab7d..00000000 --- a/src/virtualship/expedition/expedition_cost.py +++ /dev/null @@ -1,27 +0,0 @@ -"""expedition_cost function.""" - -from datetime import timedelta - -from .simulate_schedule import ScheduleOk - - -def expedition_cost(schedule_results: ScheduleOk, time_past: timedelta) -> float: - """ - Calculate the cost of the expedition in US$. - - :param schedule_results: Results from schedule simulation. - :param time_past: Time the expedition took. - :returns: The calculated cost of the expedition in US$. - """ - SHIP_COST_PER_DAY = 30000 - DRIFTER_DEPLOY_COST = 2500 - ARGO_DEPLOY_COST = 15000 - - ship_cost = SHIP_COST_PER_DAY / 24 * time_past.total_seconds() // 3600 - num_argos = len(schedule_results.measurements_to_simulate.argo_floats) - argo_cost = num_argos * ARGO_DEPLOY_COST - num_drifters = len(schedule_results.measurements_to_simulate.drifters) - drifter_cost = num_drifters * DRIFTER_DEPLOY_COST - - cost = ship_cost + argo_cost + drifter_cost - return cost diff --git a/src/virtualship/expedition/checkpoint.py b/src/virtualship/models/checkpoint.py similarity index 100% rename from src/virtualship/expedition/checkpoint.py rename to src/virtualship/models/checkpoint.py diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 8f1fadd4..af98cd3d 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -15,8 +15,10 @@ from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: + from virtualship.expedition.simulate_schedule import ScheduleOk from virtualship.models import Expedition + import pandas as pd import yaml from pydantic import BaseModel @@ -417,3 +419,25 @@ def _start_end_in_product_timerange( np.datetime64(schedule_start) >= time_min and np.datetime64(schedule_end) <= time_max ) + + +def expedition_cost(schedule_results: ScheduleOk, time_past: timedelta) -> float: + """ + Calculate the cost of the expedition in US$. + + :param schedule_results: Results from schedule simulation. + :param time_past: Time the expedition took. + :returns: The calculated cost of the expedition in US$. + """ + SHIP_COST_PER_DAY = 30000 + DRIFTER_DEPLOY_COST = 2500 + ARGO_DEPLOY_COST = 15000 + + ship_cost = SHIP_COST_PER_DAY / 24 * time_past.total_seconds() // 3600 + num_argos = len(schedule_results.measurements_to_simulate.argo_floats) + argo_cost = num_argos * ARGO_DEPLOY_COST + num_drifters = len(schedule_results.measurements_to_simulate.drifters) + drifter_cost = num_drifters * DRIFTER_DEPLOY_COST + + cost = ship_cost + argo_cost + drifter_cost + return cost From d1acdef6d88cf91f3842e47d6022256efe50dce3 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 6 Nov 2025 09:48:40 +0100 Subject: [PATCH 52/97] update imports for expedition_cost --- src/virtualship/cli/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index 46815166..d364dae5 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -8,7 +8,6 @@ import pyproj from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash -from virtualship.expedition.expedition_cost import expedition_cost from virtualship.expedition.simulate_schedule import ( MeasurementsToSimulate, ScheduleProblem, @@ -19,6 +18,7 @@ from virtualship.utils import ( CHECKPOINT, _get_expedition, + expedition_cost, get_instrument_class, ) From d7531ae416131a3e1bba0a9cf6ac9b6e75943658 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 6 Nov 2025 14:14:10 +0100 Subject: [PATCH 53/97] working with drifters (bodge), CTD_BGC not yet working --- src/virtualship/instruments/base.py | 97 ++++++++++++++++++++++---- src/virtualship/instruments/ctd_bgc.py | 2 + src/virtualship/utils.py | 4 ++ 3 files changed, 89 insertions(+), 14 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index a0e34e42..9f3e9f53 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -4,12 +4,19 @@ from typing import TYPE_CHECKING import copernicusmarine +import numpy as np import xarray as xr from yaspin import yaspin from parcels import Field, FieldSet from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash -from virtualship.utils import _get_bathy_data, _select_product_id, ship_spinner +from virtualship.utils import ( + COPERNICUSMARINE_BGC_VARIABLES, + COPERNICUSMARINE_PHYS_VARIABLES, + _get_bathy_data, + _select_product_id, + ship_spinner, +) if TYPE_CHECKING: from virtualship.models import Expedition, SpaceTimeRegion @@ -140,15 +147,25 @@ def load_input_data(self) -> FieldSet: try: datasets = [] for var in self.variables.values(): - physical = ( - True if var in ("uo", "vo", "so", "thetao") else False - ) # TODO: add more if start using new physical variables! Or more dynamic way of determining? - ds = self._get_copernicus_ds( - physical=physical, var=var - ) # user should be prompted for credentials + physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False + + # TODO: TEMPORARY BODGE FOR DRIFTER INSTRUMENT - REMOVE WHEN ABLE TO! + if self.name == "Drifter": + ds = self._get_copernicus_ds_DRIFTER(physical=physical, var=var) + else: + ds = self._get_copernicus_ds(physical=physical, var=var) datasets.append(ds) + # make sure time dims are matched if BGC variables are present (different monthly/daily resolutions can impact fieldset_endtime in simulate) + if any( + key in COPERNICUSMARINE_BGC_VARIABLES + for key in ds.keys() + for ds in datasets + ): + datasets = self._align_temporal(datasets) + ds_concat = xr.merge(datasets) # TODO: deal with WARNINGS? + fieldset = FieldSet.from_xarray_dataset( ds_concat, self.variables, self.dimensions, mesh="spherical" ) @@ -200,9 +217,6 @@ def load_input_data(self) -> FieldSet: bathymetry_field.data = -bathymetry_field.data fieldset.add_field(bathymetry_field) - # TODO: is this line necessary?! - # fieldset.computeTimeChunk(0, 1) # read in data already - return fieldset @abc.abstractmethod @@ -211,8 +225,6 @@ def simulate(self, data_dir: Path, measurements: list, out_path: str | Path): def run(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - # TODO: this will have to be able to handle the non-spinner/instead progress bar for drifters and argos! - if not self.verbose_progress: with yaspin( text=f"Simulating {self.name} measurements... ", @@ -226,6 +238,8 @@ def run(self, measurements: list, out_path: str | Path) -> None: self.simulate(measurements, out_path) print("\n") + # self.simulate(measurements, out_path) + def _get_data_dir(self, expedition_dir: Path) -> Path: space_time_region_hash = get_space_time_region_hash( self.expedition.schedule.space_time_region @@ -238,7 +252,11 @@ def _get_data_dir(self, expedition_dir: Path) -> Path: return data_dir - def _get_copernicus_ds(self, physical: bool, var: str) -> xr.Dataset: + def _get_copernicus_ds( + self, + physical: bool, + var: str, + ) -> xr.Dataset: """Get Copernicus Marine dataset for direct ingestion.""" product_id = _select_product_id( physical=physical, @@ -254,8 +272,59 @@ def _get_copernicus_ds(self, physical: bool, var: str) -> xr.Dataset: maximum_longitude=self.expedition.schedule.space_time_region.spatial_range.maximum_longitude, minimum_latitude=self.expedition.schedule.space_time_region.spatial_range.minimum_latitude, maximum_latitude=self.expedition.schedule.space_time_region.spatial_range.maximum_latitude, - variables=["uo", "vo", "so", "thetao"], + variables=[var], start_datetime=self.expedition.schedule.space_time_region.time_range.start_time, end_datetime=self.expedition.schedule.space_time_region.time_range.end_time, coordinates_selection_method="outside", ) + + # TODO: TEMPORARY BODGE FOR DRIFTER INSTRUMENT - REMOVE WHEN ABLE TO! + def _get_copernicus_ds_DRIFTER( + self, + physical: bool, + var: str, + ) -> xr.Dataset: + """Get Copernicus Marine dataset for direct ingestion.""" + product_id = _select_product_id( + physical=physical, + schedule_start=self.expedition.schedule.space_time_region.time_range.start_time, + schedule_end=self.expedition.schedule.space_time_region.time_range.end_time, + variable=var if not physical else None, + ) + + return copernicusmarine.open_dataset( + dataset_id=product_id, + dataset_part="default", + minimum_longitude=self.expedition.schedule.space_time_region.spatial_range.minimum_longitude + - 3.0, + maximum_longitude=self.expedition.schedule.space_time_region.spatial_range.maximum_longitude + + 3.0, + minimum_latitude=self.expedition.schedule.space_time_region.spatial_range.minimum_latitude + - 3.0, + maximum_latitude=self.expedition.schedule.space_time_region.spatial_range.maximum_latitude + + 3.0, + maximum_depth=1.0, + minimum_depth=1.0, + variables=[var], + start_datetime=self.expedition.schedule.space_time_region.time_range.start_time, + end_datetime=self.expedition.schedule.space_time_region.time_range.end_time + + timedelta(days=21.0), + coordinates_selection_method="outside", + ) + + def _align_temporal(self, datasets: list[xr.Dataset]) -> list[xr.Dataset]: + """Align monthly and daily time dims of multiple datasets (by repeating monthly values daily).""" + reference_time = datasets[ + np.argmax(ds.time for ds in datasets) + ].time # daily timeseries + + datasets_aligned = [] + for ds in datasets: + if not np.array_equal(ds.time, reference_time): + # TODO: NEED TO CHOOSE BEST METHOD HERE + # ds = ds.resample(time="1D").ffill().reindex(time=reference_time) + # ds = ds.resample(time="1D").ffill() + ds = ds.reindex({"time": reference_time}, method="nearest") + datasets_aligned.append(ds) + + return datasets_aligned diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 89173bbf..505e4bcc 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -272,6 +272,8 @@ def simulate(self, measurements, out_path) -> None: # define output file for the simulation out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=OUTPUT_DT) + breakpoint() + # execute simulation ctd_bgc_particleset.execute( [ diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index a013d37e..19a6d30b 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -333,6 +333,10 @@ def add_dummy_UV(fieldset: FieldSet): "phyc": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", } +# variables used in VirtualShip which are physical or biogeochemical variables, respectively +COPERNICUSMARINE_PHYS_VARIABLES = ["uo", "vo", "so", "thetao"] +COPERNICUSMARINE_BGC_VARIABLES = ["o2", "chl", "no3", "po4", "ph", "phyc", "nppv"] + def _select_product_id( physical: bool, From 118e685f60c0e7e8f9d42894953ce1ec707b1be8 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 7 Nov 2025 09:18:42 +0100 Subject: [PATCH 54/97] remove fetch and all associated logic --- src/virtualship/cli/_creds.py | 5 + src/virtualship/cli/_fetch.py | 240 ------------------ src/virtualship/cli/_run.py | 16 +- src/virtualship/cli/commands.py | 41 +-- src/virtualship/cli/main.py | 1 - src/virtualship/instruments/adcp.py | 46 +--- src/virtualship/instruments/argo_float.py | 57 +---- src/virtualship/instruments/base.py | 197 +++----------- src/virtualship/instruments/ctd.py | 50 +--- src/virtualship/instruments/ctd_bgc.py | 75 +----- src/virtualship/instruments/drifter.py | 52 +--- .../instruments/ship_underwater_st.py | 52 +--- src/virtualship/instruments/xbt.py | 57 +---- src/virtualship/models/expedition.py | 35 +-- src/virtualship/utils.py | 15 +- 15 files changed, 76 insertions(+), 863 deletions(-) delete mode 100644 src/virtualship/cli/_fetch.py diff --git a/src/virtualship/cli/_creds.py b/src/virtualship/cli/_creds.py index 9f1d2435..d9169ec4 100644 --- a/src/virtualship/cli/_creds.py +++ b/src/virtualship/cli/_creds.py @@ -1,3 +1,8 @@ +# + + +# TODO: TO DELETE?! + from __future__ import annotations from pathlib import Path diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py deleted file mode 100644 index 9722fe36..00000000 --- a/src/virtualship/cli/_fetch.py +++ /dev/null @@ -1,240 +0,0 @@ -from __future__ import annotations - -import hashlib -import shutil -from datetime import datetime -from pathlib import Path -from typing import TYPE_CHECKING - -import copernicusmarine -from copernicusmarine.core_functions.credentials_utils import InvalidUsernameOrPassword -from pydantic import BaseModel - -from virtualship.errors import IncompleteDownloadError -from virtualship.utils import ( - _dump_yaml, - _generic_load_yaml, - _get_expedition, - get_input_dataset_class, -) - -if TYPE_CHECKING: - from virtualship.models import SpaceTimeRegion - -import click - -import virtualship.cli._creds as creds -from virtualship.utils import EXPEDITION - -DOWNLOAD_METADATA = "download_metadata.yaml" - - -def _fetch(path: str | Path, username: str | None, password: str | None) -> None: - """ - Download input data for an expedition. - - Entrypoint for the tool to download data based on space-time region provided in the - schedule file. Data is downloaded from Copernicus Marine, credentials for which can be - obtained via registration: https://data.marine.copernicus.eu/register . Credentials can - be provided on prompt, via command line arguments, or via a YAML config file. Run - `virtualship fetch` on an expedition for more info. - """ - # ################################# TEMPORARY TIMER: START ################################# - import time - - start_time = time.time() - print("[TIMER] Expedition started...") - # ################################# TEMPORARY TIMER: START ################################# - - if sum([username is None, password is None]) == 1: - raise ValueError("Both username and password must be provided when using CLI.") - - path = Path(path) - - data_dir = path / "data" - data_dir.mkdir(exist_ok=True) - - expedition = _get_expedition(path) - - expedition.schedule.verify( - expedition.ship_config.ship_speed_knots, - bathy_data_dir=None, - check_space_time_region=True, - ignore_missing_bathymetry=True, - ) - - space_time_region_hash = get_space_time_region_hash( - expedition.schedule.space_time_region - ) - - existing_download = get_existing_download(data_dir, space_time_region_hash) - if existing_download is not None: - click.echo( - f"Data download for space-time region already completed ('{existing_download}')." - ) - return - - creds_path = path / creds.CREDENTIALS_FILE - credentials = {} - credentials["username"], credentials["password"] = creds.get_credentials_flow( - username, password, creds_path - ) - - # Extract instruments and space_time_region details from expedition - instruments_in_expedition = expedition.get_instruments() - space_time_region = expedition.schedule.space_time_region - - # Create download folder and set download metadata - download_folder = data_dir / hash_to_filename(space_time_region_hash) - download_folder.mkdir() - DownloadMetadata(download_complete=False).to_yaml( - download_folder / DOWNLOAD_METADATA - ) - shutil.copyfile(path / EXPEDITION, download_folder / EXPEDITION) - - click.echo(f"\n\n{(' Fetching data for: Bathymetry ').center(80, '=')}\n\n") - - # bathymetry (for all expeditions) - copernicusmarine.subset( - dataset_id="cmems_mod_glo_phy_my_0.083deg_static", - variables=["deptho"], - minimum_longitude=space_time_region.spatial_range.minimum_longitude, - maximum_longitude=space_time_region.spatial_range.maximum_longitude, - minimum_latitude=space_time_region.spatial_range.minimum_latitude, - maximum_latitude=space_time_region.spatial_range.maximum_latitude, - start_datetime=space_time_region.time_range.start_time, - end_datetime=space_time_region.time_range.start_time, - minimum_depth=abs(space_time_region.spatial_range.minimum_depth), - maximum_depth=abs(space_time_region.spatial_range.maximum_depth), - output_filename="bathymetry.nc", - output_directory=download_folder, - username=credentials["username"], - password=credentials["password"], - overwrite=True, - coordinates_selection_method="outside", - ) - - # download, only instruments present in the expedition - for itype in instruments_in_expedition: - input_dataset_class = get_input_dataset_class(itype) - if input_dataset_class is None: - raise RuntimeError(f"No input dataset class found for type {itype}.") - click.echo( - f"\n\n{(' Fetching data for: ' + itype.value + ' ').center(80, '=')}\n\n" - ) - try: - input_dataset = input_dataset_class( - data_dir=download_folder, - credentials=credentials, - space_time_region=space_time_region, - ) - input_dataset.download_data() - except InvalidUsernameOrPassword as e: - shutil.rmtree(download_folder) - raise e - click.echo(f"{itype.value} data download completed.") - - complete_download(download_folder) - - ################################# TEMPORARY TIMER: END ################################# - end_time = time.time() - elapsed = end_time - start_time - print(f"[TIMER] Expedition completed in {elapsed:.2f} seconds.") - ################################# TEMPORARY TIMER: END ################################# - - -def _hash(s: str, *, length: int) -> str: - """Create a hash of a string.""" - assert length % 2 == 0, "Length must be even." - half_length = length // 2 - - return hashlib.shake_128(s.encode("utf-8")).hexdigest(half_length) - - -def create_hash(s: str) -> str: - """Create an 8 digit hash of a string.""" - return _hash(s, length=8) - - -def hash_model(model: BaseModel, salt: int = 0) -> str: - """ - Hash a Pydantic model. - - :param region: The region to hash. - :param salt: Salt to add to the hash. - :returns: The hash. - """ - return create_hash(model.model_dump_json() + str(salt)) - - -def get_space_time_region_hash(space_time_region: SpaceTimeRegion) -> str: - # Increment salt in the event of breaking data fetching changes with prior versions - # of virtualship where you want to force new hashes (i.e., new data downloads) - salt = 0 - return hash_model(space_time_region, salt=salt) - - -def filename_to_hash(filename: str) -> str: - """Extract hash from filename of the format YYYYMMDD_HHMMSS_{hash}.""" - parts = filename.split("_") - if len(parts) != 3: - raise ValueError( - f"Filename '{filename}' must have 3 parts delimited with underscores." - ) - return parts[-1] - - -def hash_to_filename(hash: str) -> str: - """Return a filename of the format YYYYMMDD_HHMMSS_{hash}.""" - if "_" in hash: - raise ValueError("Hash cannot contain underscores.") - return f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{hash}" - - -class DownloadMetadata(BaseModel): - """Metadata for a data download.""" - - download_complete: bool - download_date: datetime | None = None - - def to_yaml(self, file_path: str | Path) -> None: - with open(file_path, "w") as file: - _dump_yaml(self, file) - - @classmethod - def from_yaml(cls, file_path: str | Path) -> DownloadMetadata: - return _generic_load_yaml(file_path, cls) - - -def get_existing_download(data_dir: Path, space_time_region_hash: str) -> Path | None: - """Check if a download has already been completed. If so, return the path for existing download.""" - for download_path in data_dir.rglob("*"): - try: - hash = filename_to_hash(download_path.name) - except ValueError: - continue - if hash == space_time_region_hash: - assert_complete_download(download_path) - return download_path - return None - - -def assert_complete_download(download_path: Path) -> None: - download_metadata = download_path / DOWNLOAD_METADATA - try: - with open(download_metadata) as file: - assert DownloadMetadata.from_yaml(file).download_complete - except (FileNotFoundError, AssertionError) as e: - raise IncompleteDownloadError( - f"Download at {download_path} was found, but looks to be incomplete " - f"(likely due to interupting it mid-download). Please delete this folder and retry." - ) from e - return - - -def complete_download(download_path: Path) -> None: - """Mark a download as complete.""" - download_metadata = download_path / DOWNLOAD_METADATA - metadata = DownloadMetadata(download_complete=True, download_date=datetime.now()) - metadata.to_yaml(download_metadata) - return diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index 4eb50c66..df8a24fd 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -7,7 +7,6 @@ import pyproj -from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash from virtualship.expedition.simulate_schedule import ( MeasurementsToSimulate, ScheduleProblem, @@ -34,12 +33,11 @@ logging.getLogger("copernicusmarine").setLevel("ERROR") -def _run(expedition_dir: str | Path, from_copernicusmarine: bool) -> None: +def _run(expedition_dir: str | Path) -> None: """ Perform an expedition, providing terminal feedback and file output. :param expedition_dir: The base directory for the expedition. - :param from_copernicusmarine: Whether to use direct data ingestion from Copernicus Marine. Should be determined by CLI flag. """ # ################################# TEMPORARY TIMER: START ################################# import time @@ -70,16 +68,7 @@ def _run(expedition_dir: str | Path, from_copernicusmarine: bool) -> None: print("\n---- WAYPOINT VERIFICATION ----") - # verify schedule is valid - if from_copernicusmarine: - bathy_data_dir = None - else: - bathy_data_dir = get_existing_download( - expedition_dir, - get_space_time_region_hash(expedition.schedule.space_time_region), - ) - - expedition.schedule.verify(expedition.ship_config.ship_speed_knots, bathy_data_dir) + expedition.schedule.verify(expedition.ship_config.ship_speed_knots) # simulate the schedule schedule_results = simulate_schedule( @@ -133,7 +122,6 @@ def _run(expedition_dir: str | Path, from_copernicusmarine: bool) -> None: instrument = instrument_class( expedition=expedition, directory=expedition_dir, - from_copernicusmarine=from_copernicusmarine, ) # execute simulation diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index e2a4aade..b17f3cbc 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -3,7 +3,6 @@ import click from virtualship import utils -from virtualship.cli._fetch import _fetch from virtualship.cli._plan import _plan from virtualship.cli._run import _run from virtualship.utils import ( @@ -76,48 +75,12 @@ def plan(path): _plan(Path(path)) -@click.command() -@click.argument( - "path", - type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True), -) -@click.option( - "--username", - type=str, - default=None, - help="Copernicus Marine username.", -) -@click.option( - "--password", - type=str, - default=None, - help="Copernicus Marine password.", -) -def fetch(path: str | Path, username: str | None, password: str | None) -> None: - """ - Download input data for an expedition. - - Entrypoint for the tool to download data based on space-time region provided in the - schedule file. Data is downloaded from Copernicus Marine, credentials for which can be - obtained via registration: https://data.marine.copernicus.eu/register . Credentials can - be provided on prompt, via command line arguments, or via a YAML config file. Run - `virtualship fetch` on a expedition for more info. - """ - _fetch(path, username, password) - - # TODO: also add option to 'stream' via link to dir elsewhere, e.g. simlink or path to data stored elsewhere that isn't expedition dir! @click.command() @click.argument( "path", type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True), ) -@click.option( - "--from-copernicusmarine", - is_flag=True, - default=False, - help="Ingest fieldsets directly via copernicusmarine toolbox.", -) -def run(path, from_copernicusmarine: bool): +def run(path): """Run the expedition.""" - _run(Path(path), from_copernicusmarine) + _run(Path(path)) diff --git a/src/virtualship/cli/main.py b/src/virtualship/cli/main.py index 6ee12eff..a02a5ffb 100644 --- a/src/virtualship/cli/main.py +++ b/src/virtualship/cli/main.py @@ -11,7 +11,6 @@ def cli(): cli.add_command(commands.init) cli.add_command(commands.plan) -cli.add_command(commands.fetch) cli.add_command(commands.run) if __name__ == "__main__": diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 2702cbfd..d01b675b 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -4,10 +4,9 @@ import numpy as np from parcels import ParticleSet, ScipyParticle, Variable -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( - register_input_dataset, register_instrument, ) @@ -46,46 +45,6 @@ def _sample_velocity(particle, fieldset, time): ) -# ===================================================== -# SECTION: InputDataset Class -# ===================================================== - - -@register_input_dataset(InstrumentType.ADCP) -class ADCPInputDataset(InputDataset): - """Input dataset for ADCP instrument.""" - - DOWNLOAD_BUFFERS: ClassVar[dict] = { - "latlon_degrees": 0.0, - "days": 0.0, - } # ADCP data requires no buffers - - DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} - - def __init__(self, data_dir, credentials, space_time_region): - """Initialise with instrument's name.""" - super().__init__( - ADCP.name, - self.DOWNLOAD_BUFFERS["latlon_degrees"], - self.DOWNLOAD_BUFFERS["days"], - space_time_region.spatial_range.minimum_depth, - space_time_region.spatial_range.maximum_depth, - data_dir, - credentials, - space_time_region, - ) - - def get_datasets_dict(self) -> dict: - """Get variable specific args for instrument.""" - return { - "UVdata": { - "physical": True, - "variables": ["uo", "vo"], - "output_filename": f"{self.name}_uv.nc", - }, - } - - # ===================================================== # SECTION: Instrument Class # ===================================================== @@ -95,7 +54,7 @@ def get_datasets_dict(self) -> dict: class ADCPInstrument(Instrument): """ADCP instrument class.""" - def __init__(self, expedition, directory, from_copernicusmarine): + def __init__(self, expedition, directory): """Initialize ADCPInstrument.""" filenames = { "U": f"{ADCP.name}_uv.nc", @@ -111,7 +70,6 @@ def __init__(self, expedition, directory, from_copernicusmarine): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, - from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 30def124..ce539d6b 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -12,10 +12,10 @@ StatusCode, Variable, ) -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_input_dataset, register_instrument +from virtualship.utils import register_instrument # ===================================================== # SECTION: Dataclass @@ -128,56 +128,6 @@ def _check_error(particle, fieldset, time): particle.delete() -# ===================================================== -# SECTION: InputDataset Class -# ===================================================== - - -@register_input_dataset(InstrumentType.ARGO_FLOAT) -class ArgoFloatInputDataset(InputDataset): - """Input dataset for ArgoFloat instrument.""" - - DOWNLOAD_BUFFERS: ClassVar[dict] = { - "latlon_degrees": 3.0, - "days": 21.0, - } - - DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} - - def __init__(self, data_dir, credentials, space_time_region): - """Initialise with instrument's name.""" - super().__init__( - ArgoFloat.name, - self.DOWNLOAD_BUFFERS["latlon_degrees"], - self.DOWNLOAD_BUFFERS["days"], - self.DOWNLOAD_LIMITS["min_depth"], - space_time_region.spatial_range.maximum_depth, - data_dir, - credentials, - space_time_region, - ) - - def get_datasets_dict(self) -> dict: - """Get variable specific args for instrument.""" - return { - "UVdata": { - "physical": True, - "variables": ["uo", "vo"], - "output_filename": f"{self.name}_uv.nc", - }, - "Sdata": { - "physical": True, - "variables": ["so"], - "output_filename": f"{self.name}_s.nc", - }, - "Tdata": { - "physical": True, - "variables": ["thetao"], - "output_filename": f"{self.name}_t.nc", - }, - } - - # ===================================================== # SECTION: Instrument Class # ===================================================== @@ -187,7 +137,7 @@ def get_datasets_dict(self) -> dict: class ArgoFloatInstrument(Instrument): """ArgoFloat instrument class.""" - def __init__(self, expedition, directory, from_copernicusmarine): + def __init__(self, expedition, directory): """Initialize ArgoFloatInstrument.""" filenames = { "U": f"{ArgoFloat.name}_uv.nc", @@ -205,7 +155,6 @@ def __init__(self, expedition, directory, from_copernicusmarine): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, - from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index b040c3db..311d83f6 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -8,8 +8,7 @@ import xarray as xr from yaspin import yaspin -from parcels import Field, FieldSet -from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash +from parcels import FieldSet from virtualship.utils import ( COPERNICUSMARINE_BGC_VARIABLES, COPERNICUSMARINE_PHYS_VARIABLES, @@ -19,86 +18,7 @@ ) if TYPE_CHECKING: - from virtualship.models import Expedition, SpaceTimeRegion - - -class InputDataset(abc.ABC): - """Base class for instrument input datasets.""" - - # TODO: data download is performed per instrument (in `fetch`), which is a bit inefficient when some instruments can share dataa. - # TODO: However, future changes, with Parcels-v4 and copernicusmarine direct ingestion, will hopefully remove the need for fetch. - - def __init__( - self, - name: str, - latlon_buffer: float, - datetime_buffer: float, - min_depth: float, - max_depth: float, - data_dir: str, - credentials: dict, - space_time_region: "SpaceTimeRegion", - ): - """Initialise input dataset.""" - self.name = name - self.latlon_buffer = latlon_buffer - self.datetime_buffer = datetime_buffer - self.min_depth = min_depth - self.max_depth = max_depth - self.data_dir = data_dir - self.credentials = credentials - self.space_time_region = space_time_region - - @abc.abstractmethod - def get_datasets_dict(self) -> dict: - """Get parameters for instrument's variable(s) specific data download.""" - - def download_data(self) -> None: - """Download data for the instrument using copernicusmarine, with correct product ID selection.""" - parameter_args = dict( - minimum_longitude=self.space_time_region.spatial_range.minimum_longitude - - self.latlon_buffer, - maximum_longitude=self.space_time_region.spatial_range.maximum_longitude - + self.latlon_buffer, - minimum_latitude=self.space_time_region.spatial_range.minimum_latitude - - self.latlon_buffer, - maximum_latitude=self.space_time_region.spatial_range.maximum_latitude - + self.latlon_buffer, - start_datetime=self.space_time_region.time_range.start_time, - end_datetime=self.space_time_region.time_range.end_time - + timedelta(days=self.datetime_buffer), - minimum_depth=abs(self.min_depth), - maximum_depth=abs(self.max_depth), - output_directory=self.data_dir, - username=self.credentials["username"], - password=self.credentials["password"], - overwrite=True, - coordinates_selection_method="outside", - ) - - datasets_args = self.get_datasets_dict() - - for dataset in datasets_args.values(): - physical = dataset.get("physical") - if physical: - variable = None - else: - variable = dataset.get("variables")[0] # BGC variables, special case - - dataset_id = _select_product_id( - physical=physical, - schedule_start=self.space_time_region.time_range.start_time, - schedule_end=self.space_time_region.time_range.end_time, - username=self.credentials["username"], - password=self.credentials["password"], - variable=variable, - ) - download_args = { - **parameter_args, - **{k: v for k, v in dataset.items() if k != "physical"}, - "dataset_id": dataset_id, - } - copernicusmarine.subset(**download_args) + from virtualship.models import Expedition class Instrument(abc.ABC): @@ -107,7 +27,6 @@ class Instrument(abc.ABC): #! TODO List: # TODO: update documentation/quickstart # TODO: update tests - # TODO: if use direct ingestion as primary data sourcing, can substantially cut code base (including _fetch.py, InputDataset objects). Consider this for Parcels v4 transition. #! TODO: how is this handling credentials?! Seems to work already, are these set up from my previous instances of using copernicusmarine? Therefore users will only have to do it once too? def __init__( @@ -121,7 +40,6 @@ def __init__( allow_time_extrapolation: bool, verbose_progress: bool, bathymetry_file: str = "bathymetry.nc", - from_copernicusmarine: bool = False, ): """Initialise instrument.""" self.name = name @@ -139,61 +57,40 @@ def __init__( self.add_bathymetry = add_bathymetry self.allow_time_extrapolation = allow_time_extrapolation self.verbose_progress = verbose_progress - self.from_copernicusmarine = from_copernicusmarine def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" - if self.from_copernicusmarine: - try: - datasets = [] - for var in self.variables.values(): - physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False - - # TODO: TEMPORARY BODGE FOR DRIFTER INSTRUMENT - REMOVE WHEN ABLE TO! - if self.name == "Drifter": - ds = self._get_copernicus_ds_DRIFTER(physical=physical, var=var) - else: - ds = self._get_copernicus_ds(physical=physical, var=var) - datasets.append(ds) - - # make sure time dims are matched if BGC variables are present (different monthly/daily resolutions can impact fieldset_endtime in simulate) - if any( - key in COPERNICUSMARINE_BGC_VARIABLES - for key in ds.keys() - for ds in datasets - ): - datasets = self._align_temporal(datasets) - - ds_concat = xr.merge(datasets) # TODO: deal with WARNINGS? - - fieldset = FieldSet.from_xarray_dataset( - ds_concat, self.variables, self.dimensions, mesh="spherical" - ) - - except Exception as e: - raise FileNotFoundError( - f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'. " - f"Please check your credentials, network connection, and variable names. Original error: {e}" - ) from e + try: + datasets = [] + for var in self.variables.values(): + physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False + + # TODO: TEMPORARY BODGE FOR DRIFTER INSTRUMENT - REMOVE WHEN ABLE TO! + if self.name == "Drifter": + ds = self._get_copernicus_ds_DRIFTER(physical=physical, var=var) + else: + ds = self._get_copernicus_ds(physical=physical, var=var) + datasets.append(ds) + + # make sure time dims are matched if BGC variables are present (different monthly/daily resolutions can impact fieldset_endtime in simulate) + if any( + key in COPERNICUSMARINE_BGC_VARIABLES + for key in ds.keys() + for ds in datasets + ): + datasets = self._align_temporal(datasets) + + ds_concat = xr.merge(datasets) # TODO: deal with WARNINGS? + + fieldset = FieldSet.from_xarray_dataset( + ds_concat, self.variables, self.dimensions, mesh="spherical" + ) - else: # from fetched data on disk - try: - data_dir = self._get_data_dir(self.directory) - joined_filepaths = { - key: data_dir.joinpath(filename) - for key, filename in self.filenames.items() - } - fieldset = FieldSet.from_netcdf( - joined_filepaths, - self.variables, - self.dimensions, - allow_time_extrapolation=self.allow_time_extrapolation, - ) - except FileNotFoundError as e: - raise FileNotFoundError( - f"Input data for instrument {self.name} not found locally. Have you run the `virtualship fetch` command?" - "Alternatively, you can use the `--from-copernicusmarine` option to ingest data directly from Copernicus Marine." - ) from e + except Exception as e: + raise FileNotFoundError( + f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'. " + f"Please check your credentials, network connection, and variable names. Original error: {e}" + ) from e # interpolation methods for var in (v for v in self.variables if v not in ("U", "V")): @@ -203,19 +100,11 @@ def load_input_data(self) -> FieldSet: g.negate_depth() # bathymetry data - if self.add_bathymetry: - if self.from_copernicusmarine: - bathymetry_field = _get_bathy_data( - self.expedition.schedule.space_time_region - ).bathymetry - else: - bathymetry_field = Field.from_netcdf( - data_dir.joinpath(self.bathymetry_file), - variable=("bathymetry", "deptho"), - dimensions={"lon": "longitude", "lat": "latitude"}, - ) - bathymetry_field.data = -bathymetry_field.data - fieldset.add_field(bathymetry_field) + bathymetry_field = _get_bathy_data( + self.expedition.schedule.space_time_region + ).bathymetry + bathymetry_field.data = -bathymetry_field.data + fieldset.add_field(bathymetry_field) return fieldset @@ -240,18 +129,6 @@ def execute(self, measurements: list, out_path: str | Path) -> None: # self.simulate(measurements, out_path) - def _get_data_dir(self, expedition_dir: Path) -> Path: - space_time_region_hash = get_space_time_region_hash( - self.expedition.schedule.space_time_region - ) - data_dir = get_existing_download(expedition_dir, space_time_region_hash) - - assert data_dir is not None, ( - "Input data hasn't been found. Have you run the `virtualship fetch` command?" - ) - - return data_dir - def _get_copernicus_ds( self, physical: bool, diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 7434da2d..d205795b 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -5,12 +5,12 @@ import numpy as np from parcels import JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType if TYPE_CHECKING: from virtualship.models.spacetime import Spacetime -from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument +from virtualship.utils import add_dummy_UV, register_instrument # ===================================================== # SECTION: Dataclass @@ -70,49 +70,6 @@ def _ctd_cast(particle, fieldset, time): particle.delete() -# ===================================================== -# SECTION: InputDataset Class -# ===================================================== - - -@register_input_dataset(InstrumentType.CTD) -class CTDInputDataset(InputDataset): - """Input dataset for CTD instrument.""" - - DOWNLOAD_BUFFERS: ClassVar[dict] = { - "latlon_degrees": 0.0, - "days": 0.0, - } # CTD data requires no buffers - - def __init__(self, data_dir, credentials, space_time_region): - """Initialise with instrument's name.""" - super().__init__( - CTD.name, - self.DOWNLOAD_BUFFERS["latlon_degrees"], - self.DOWNLOAD_BUFFERS["days"], - space_time_region.spatial_range.minimum_depth, - space_time_region.spatial_range.maximum_depth, - data_dir, - credentials, - space_time_region, - ) - - def get_datasets_dict(self) -> dict: - """Get variable specific args for instrument.""" - return { - "Sdata": { - "physical": True, - "variables": ["so"], - "output_filename": f"{self.name}_s.nc", - }, - "Tdata": { - "physical": True, - "variables": ["thetao"], - "output_filename": f"{self.name}_t.nc", - }, - } - - # ===================================================== # SECTION: Instrument Class # ===================================================== @@ -122,7 +79,7 @@ def get_datasets_dict(self) -> dict: class CTDInstrument(Instrument): """CTD instrument class.""" - def __init__(self, expedition, directory, from_copernicusmarine): + def __init__(self, expedition, directory): """Initialize CTDInstrument.""" filenames = { "S": f"{CTD.name}_s.nc", @@ -139,7 +96,6 @@ def __init__(self, expedition, directory, from_copernicusmarine): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, - from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 505e4bcc..182d274c 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -5,10 +5,10 @@ import numpy as np from parcels import JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument +from virtualship.utils import add_dummy_UV, register_instrument # ===================================================== # SECTION: Dataclass @@ -92,74 +92,6 @@ def _ctd_bgc_cast(particle, fieldset, time): particle.delete() -# ===================================================== -# SECTION: InputDataset Class -# ===================================================== - - -@register_input_dataset(InstrumentType.CTD_BGC) -class CTD_BGCInputDataset(InputDataset): - """Input dataset object for CTD_BGC instrument.""" - - DOWNLOAD_BUFFERS: ClassVar[dict] = { - "latlon_degrees": 0.0, - "days": 0.0, - } # CTD_BGC data requires no buffers - - def __init__(self, data_dir, credentials, space_time_region): - """Initialise with instrument's name.""" - super().__init__( - CTD_BGC.name, - self.DOWNLOAD_BUFFERS["latlon_degrees"], - self.DOWNLOAD_BUFFERS["days"], - space_time_region.spatial_range.minimum_depth, - space_time_region.spatial_range.maximum_depth, - data_dir, - credentials, - space_time_region, - ) - - def get_datasets_dict(self) -> dict: - """Variable specific args for instrument.""" - return { - "o2data": { - "physical": False, - "variables": ["o2"], - "output_filename": f"{self.name}_o2.nc", - }, - "chlorodata": { - "physical": False, - "variables": ["chl"], - "output_filename": f"{self.name}_chl.nc", - }, - "nitratedata": { - "physical": False, - "variables": ["no3"], - "output_filename": f"{self.name}_no3.nc", - }, - "phosphatedata": { - "physical": False, - "variables": ["po4"], - "output_filename": f"{self.name}_po4.nc", - }, - "phdata": { - "physical": False, - "variables": ["ph"], - "output_filename": f"{self.name}_ph.nc", - }, - "phytoplanktondata": { - "physical": False, - "variables": ["phyc"], - "output_filename": f"{self.name}_phyc.nc", - }, - "primaryproductiondata": { - "physical": False, - "variables": ["nppv"], - "output_filename": f"{self.name}_nppv.nc", - }, - } - - # ===================================================== # SECTION: Instrument Class # ===================================================== @@ -169,7 +101,7 @@ def get_datasets_dict(self) -> dict: class CTD_BGCInstrument(Instrument): """CTD_BGC instrument class.""" - def __init__(self, expedition, directory, from_copernicusmarine): + def __init__(self, expedition, directory): """Initialize CTD_BGCInstrument.""" filenames = { "o2": f"{CTD_BGC.name}_o2.nc", @@ -198,7 +130,6 @@ def __init__(self, expedition, directory, from_copernicusmarine): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, - from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 182100c2..06f6b71f 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -5,10 +5,10 @@ import numpy as np from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_input_dataset, register_instrument +from virtualship.utils import register_instrument # ===================================================== # SECTION: Dataclass @@ -54,51 +54,6 @@ def _check_lifetime(particle, fieldset, time): particle.delete() -# ===================================================== -# SECTION: InputDataset Class -# ===================================================== - - -@register_input_dataset(InstrumentType.DRIFTER) -class DrifterInputDataset(InputDataset): - """Input dataset for Drifter instrument.""" - - DOWNLOAD_BUFFERS: ClassVar[dict] = { - "latlon_degrees": 3.0, - "days": 21.0, - } - - DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1, "max_depth": 1} - - def __init__(self, data_dir, credentials, space_time_region): - """Initialise with instrument's name.""" - super().__init__( - Drifter.name, - self.DOWNLOAD_BUFFERS["latlon_degrees"], - self.DOWNLOAD_BUFFERS["days"], - self.DOWNLOAD_LIMITS["min_depth"], - self.DOWNLOAD_LIMITS["max_depth"], - data_dir, - credentials, - space_time_region, - ) - - def get_datasets_dict(self) -> dict: - """Get variable specific args for instrument.""" - return { - "UVdata": { - "physical": True, - "variables": ["uo", "vo"], - "output_filename": f"{self.name}_uv.nc", - }, - "Tdata": { - "physical": True, - "variables": ["thetao"], - "output_filename": f"{self.name}_t.nc", - }, - } - - # ===================================================== # SECTION: Instrument Class # ===================================================== @@ -108,7 +63,7 @@ def get_datasets_dict(self) -> dict: class DrifterInstrument(Instrument): """Drifter instrument class.""" - def __init__(self, expedition, directory, from_copernicusmarine): + def __init__(self, expedition, directory): """Initialize DrifterInstrument.""" filenames = { "U": f"{Drifter.name}_uv.nc", @@ -125,7 +80,6 @@ def __init__(self, expedition, directory, from_copernicusmarine): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, - from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 5f793d3d..f6099869 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -4,9 +4,9 @@ import numpy as np from parcels import ParticleSet, ScipyParticle, Variable -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType -from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument +from virtualship.utils import add_dummy_UV, register_instrument # ===================================================== # SECTION: Dataclass @@ -46,51 +46,6 @@ def _sample_temperature(particle, fieldset, time): particle.T = fieldset.T[time, particle.depth, particle.lat, particle.lon] -# ===================================================== -# SECTION: InputDataset Class -# ===================================================== - - -@register_input_dataset(InstrumentType.UNDERWATER_ST) -class Underwater_STInputDataset(InputDataset): - """Input dataset for Underwater_ST instrument.""" - - DOWNLOAD_BUFFERS: ClassVar[dict] = { - "latlon_degrees": 0.0, - "days": 0.0, - } # Underwater_ST data requires no buffers - - DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} - - def __init__(self, data_dir, credentials, space_time_region): - """Initialise with instrument's name.""" - super().__init__( - Underwater_ST.name, - self.DOWNLOAD_BUFFERS["latlon_degrees"], - self.DOWNLOAD_BUFFERS["days"], - -2.0, # is always at 2m depth - -2.0, # is always at 2m depth - data_dir, - credentials, - space_time_region, - ) - - def get_datasets_dict(self) -> dict: - """Get variable specific args for instrument.""" - return { - "Sdata": { - "physical": True, - "variables": ["so"], - "output_filename": f"{self.name}_s.nc", - }, - "Tdata": { - "physical": True, - "variables": ["thetao"], - "output_filename": f"{self.name}_t.nc", - }, - } - - # ===================================================== # SECTION: Instrument Class # ===================================================== @@ -100,7 +55,7 @@ def get_datasets_dict(self) -> dict: class Underwater_STInstrument(Instrument): """Underwater_ST instrument class.""" - def __init__(self, expedition, directory, from_copernicusmarine): + def __init__(self, expedition, directory): """Initialize Underwater_STInstrument.""" filenames = { "S": f"{Underwater_ST.name}_s.nc", @@ -117,7 +72,6 @@ def __init__(self, expedition, directory, from_copernicusmarine): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, - from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index ab11ed67..fd88240d 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -5,10 +5,10 @@ import numpy as np from parcels import JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_input_dataset, register_instrument +from virtualship.utils import register_instrument # ===================================================== # SECTION: Dataclass @@ -68,56 +68,6 @@ def _xbt_cast(particle, fieldset, time): particle_ddepth = particle.max_depth - particle.depth -# ===================================================== -# SECTION: InputDataset Class -# ===================================================== - - -@register_input_dataset(InstrumentType.XBT) -class XBTInputDataset(InputDataset): - """Input dataset for XBT instrument.""" - - DOWNLOAD_BUFFERS: ClassVar[dict] = { - "latlon_degrees": 3.0, - "days": 21.0, - } - - DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} - - def __init__(self, data_dir, credentials, space_time_region): - """Initialise with instrument's name.""" - super().__init__( - XBT.name, - self.DOWNLOAD_BUFFERS["latlon_degrees"], - self.DOWNLOAD_BUFFERS["days"], - self.DOWNLOAD_LIMITS["min_depth"], - space_time_region.spatial_range.maximum_depth, - data_dir, - credentials, - space_time_region, - ) - - def get_datasets_dict(self) -> dict: - """Get variable specific args for instrument.""" - return { - "UVdata": { - "physical": True, - "variables": ["uo", "vo"], - "output_filename": f"{self.name}_uv.nc", - }, - "Sdata": { - "physical": True, - "variables": ["so"], - "output_filename": f"{self.name}_s.nc", - }, - "Tdata": { - "physical": True, - "variables": ["thetao"], - "output_filename": f"{self.name}_t.nc", - }, - } - - # ===================================================== # SECTION: Instrument Class # ===================================================== @@ -127,7 +77,7 @@ def get_datasets_dict(self) -> dict: class XBTInstrument(Instrument): """XBT instrument class.""" - def __init__(self, expedition, directory, from_copernicusmarine): + def __init__(self, expedition, directory): """Initialize XBTInstrument.""" filenames = { "U": f"{XBT.name}_uv.nc", @@ -145,7 +95,6 @@ def __init__(self, expedition, directory, from_copernicusmarine): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, - from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 14d9537d..aa310a97 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -2,14 +2,12 @@ import itertools from datetime import datetime, timedelta -from pathlib import Path from typing import TYPE_CHECKING import pydantic import pyproj import yaml -from parcels import Field from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.instruments.types import InstrumentType from virtualship.utils import _get_bathy_data, _validate_numeric_mins_to_timedelta @@ -89,7 +87,6 @@ class Schedule(pydantic.BaseModel): def verify( self, ship_speed: float, - bathy_data_dir: str | Path | None, ignore_missing_bathymetry: bool = False, *, check_space_time_region: bool = False, @@ -108,7 +105,7 @@ def verify( if check_space_time_region and self.space_time_region is None: raise ScheduleError( - "space_time_region not found in schedule, please define it to fetch the data." + "space_time_region not found in schedule, please define it to proceed." ) if len(self.waypoints) == 0: @@ -133,28 +130,14 @@ def verify( # TODO: write test that checks that will flag when waypoint is on land!! [add to existing suite of fail .verify() tests in test_expedition.py] land_waypoints = [] if not ignore_missing_bathymetry: - if bathy_data_dir is None: - try: - bathymetry_field = _get_bathy_data( - self.space_time_region - ).bathymetry # via copernicusmarine - except Exception as e: - raise ScheduleError( - f"Problem loading bathymetry data (used to verify waypoints are in water) directly via copernicusmarine. \n\n original message: {e}" - ) from e - - else: - bathymetry_path = bathy_data_dir.joinpath("bathymetry.nc") - try: - bathymetry_field = Field.from_netcdf( - bathymetry_path, - variable=("bathymetry", "deptho"), - dimensions={"lon": "longitude", "lat": "latitude"}, - ) - except Exception as e: - raise ScheduleError( - f"Problem loading local bathymetry data (used to verify waypoints are in water). Have you run `virtualship fetch` command?. \n\n original message: {e}" - ) from e + try: + bathymetry_field = _get_bathy_data( + self.space_time_region + ).bathymetry # via copernicusmarine + except Exception as e: + raise ScheduleError( + f"Problem loading bathymetry data (used to verify waypoints are in water) directly via copernicusmarine. \n\n original message: {e}" + ) from e for wp_i, wp in enumerate(self.waypoints): try: diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index dc46ded7..4ea17c4a 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -249,19 +249,10 @@ def _get_expedition(expedition_dir: Path) -> Expedition: ) -# InstrumentType -> InputDataset and Instrument registry and registration utilities. -INPUT_DATASET_MAP = {} +# InstrumentType -> Instrument registry and registration utilities. INSTRUMENT_CLASS_MAP = {} -def register_input_dataset(instrument_type): - def decorator(cls): - INPUT_DATASET_MAP[instrument_type] = cls - return cls - - return decorator - - def register_instrument(instrument_type): def decorator(cls): INSTRUMENT_CLASS_MAP[instrument_type] = cls @@ -270,10 +261,6 @@ def decorator(cls): return decorator -def get_input_dataset_class(instrument_type): - return INPUT_DATASET_MAP.get(instrument_type) - - def get_instrument_class(instrument_type): return INSTRUMENT_CLASS_MAP.get(instrument_type) From 05686f2513229286bff5cbf3bee3789353dcad2a Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 7 Nov 2025 09:24:21 +0100 Subject: [PATCH 55/97] update docstrings/--help info --- src/virtualship/cli/commands.py | 4 ++-- src/virtualship/expedition/__init__.py | 8 +------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index b17f3cbc..2b0ce8e9 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -68,7 +68,7 @@ def init(path, from_mfp): ) def plan(path): """ - Launch UI to help build schedule and ship config files. + Launch UI to help build expedition configuration (YAML) file. Should you encounter any issues with using this tool, please report an issue describing the problem to the VirtualShip issue tracker at: https://github.com/OceanParcels/virtualship/issues" """ @@ -82,5 +82,5 @@ def plan(path): type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True), ) def run(path): - """Run the expedition.""" + """Execute the expedition simulations.""" _run(Path(path)) diff --git a/src/virtualship/expedition/__init__.py b/src/virtualship/expedition/__init__.py index dfa61028..7f072bbf 100644 --- a/src/virtualship/expedition/__init__.py +++ b/src/virtualship/expedition/__init__.py @@ -1,7 +1 @@ -"""Everything for simulating an expedition.""" - -from .do_expedition import do_expedition - -__all__ = [ - "do_expedition", -] +"""Simulating an expedition.""" From 10e68b383a640038804d8effe3c22129d1f85c27 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 7 Nov 2025 11:40:19 +0100 Subject: [PATCH 56/97] add buffers to drifters and argos, plus depth limits for drifters --- src/virtualship/instruments/adcp.py | 3 + src/virtualship/instruments/argo_float.py | 7 ++ src/virtualship/instruments/base.py | 85 ++++++------------- src/virtualship/instruments/ctd.py | 2 + src/virtualship/instruments/ctd_bgc.py | 2 + src/virtualship/instruments/drifter.py | 13 ++- .../instruments/ship_underwater_st.py | 2 + src/virtualship/instruments/xbt.py | 2 + src/virtualship/models/expedition.py | 5 +- src/virtualship/utils.py | 14 +-- 10 files changed, 68 insertions(+), 67 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index d01b675b..dd675b7e 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -61,6 +61,7 @@ def __init__(self, expedition, directory): "V": f"{ADCP.name}_uv.nc", } variables = {"U": "uo", "V": "vo"} + super().__init__( ADCP.name, expedition, @@ -70,6 +71,8 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, + buffer_spec=None, + limit_spec=None, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index ce539d6b..12ee8945 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -146,6 +146,11 @@ def __init__(self, expedition, directory): "T": f"{ArgoFloat.name}_t.nc", } variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} + buffer_spec = { + "latlon": 6.0, # [degrees] + "time": 21.0, # [days] + } + super().__init__( ArgoFloat.name, expedition, @@ -155,6 +160,8 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, + buffer_spec=buffer_spec, + limit_spec=None, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 311d83f6..c290f277 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -25,9 +25,7 @@ class Instrument(abc.ABC): """Base class for instruments and their simulation.""" #! TODO List: - # TODO: update documentation/quickstart - # TODO: update tests - #! TODO: how is this handling credentials?! Seems to work already, are these set up from my previous instances of using copernicusmarine? Therefore users will only have to do it once too? + # TODO: how is this handling credentials?! Seems to work already, are these set up from my previous instances of using copernicusmarine? Therefore users will only have to do it once too? def __init__( self, @@ -39,7 +37,8 @@ def __init__( add_bathymetry: bool, allow_time_extrapolation: bool, verbose_progress: bool, - bathymetry_file: str = "bathymetry.nc", + buffer_spec: dict | None = None, + limit_spec: dict | None = None, ): """Initialise instrument.""" self.name = name @@ -53,10 +52,11 @@ def __init__( "time": "time", "depth": "depth", } # same dimensions for all instruments - self.bathymetry_file = bathymetry_file self.add_bathymetry = add_bathymetry self.allow_time_extrapolation = allow_time_extrapolation self.verbose_progress = verbose_progress + self.buffer_spec = buffer_spec + self.limit_spec = limit_spec def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" @@ -64,20 +64,11 @@ def load_input_data(self) -> FieldSet: datasets = [] for var in self.variables.values(): physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False - - # TODO: TEMPORARY BODGE FOR DRIFTER INSTRUMENT - REMOVE WHEN ABLE TO! - if self.name == "Drifter": - ds = self._get_copernicus_ds_DRIFTER(physical=physical, var=var) - else: - ds = self._get_copernicus_ds(physical=physical, var=var) - datasets.append(ds) + datasets.append(self._get_copernicus_ds(physical=physical, var=var)) # make sure time dims are matched if BGC variables are present (different monthly/daily resolutions can impact fieldset_endtime in simulate) - if any( - key in COPERNICUSMARINE_BGC_VARIABLES - for key in ds.keys() - for ds in datasets - ): + all_keys = set().union(*(ds.keys() for ds in datasets)) + if all_keys & set(COPERNICUSMARINE_BGC_VARIABLES): datasets = self._align_temporal(datasets) ds_concat = xr.merge(datasets) # TODO: deal with WARNINGS? @@ -100,11 +91,15 @@ def load_input_data(self) -> FieldSet: g.negate_depth() # bathymetry data - bathymetry_field = _get_bathy_data( - self.expedition.schedule.space_time_region - ).bathymetry - bathymetry_field.data = -bathymetry_field.data - fieldset.add_field(bathymetry_field) + if self.add_bathymetry: + bathymetry_field = _get_bathy_data( + self.expedition.schedule.space_time_region, + latlon_buffer=self.buffer_spec.get("latlon") + if self.buffer_spec + else None, + ).bathymetry + bathymetry_field.data = -bathymetry_field.data + fieldset.add_field(bathymetry_field) return fieldset @@ -127,8 +122,6 @@ def execute(self, measurements: list, out_path: str | Path) -> None: self.simulate(measurements, out_path) print("\n") - # self.simulate(measurements, out_path) - def _get_copernicus_ds( self, physical: bool, @@ -142,50 +135,28 @@ def _get_copernicus_ds( variable=var if not physical else None, ) - return copernicusmarine.open_dataset( - dataset_id=product_id, - dataset_part="default", - minimum_longitude=self.expedition.schedule.space_time_region.spatial_range.minimum_longitude, - maximum_longitude=self.expedition.schedule.space_time_region.spatial_range.maximum_longitude, - minimum_latitude=self.expedition.schedule.space_time_region.spatial_range.minimum_latitude, - maximum_latitude=self.expedition.schedule.space_time_region.spatial_range.maximum_latitude, - variables=[var], - start_datetime=self.expedition.schedule.space_time_region.time_range.start_time, - end_datetime=self.expedition.schedule.space_time_region.time_range.end_time, - coordinates_selection_method="outside", - ) + latlon_buffer = self.buffer_spec.get("latlon") if self.buffer_spec else 0.0 + time_buffer = self.buffer_spec.get("time") if self.buffer_spec else 0.0 - # TODO: TEMPORARY BODGE FOR DRIFTER INSTRUMENT - REMOVE WHEN ABLE TO! - def _get_copernicus_ds_DRIFTER( - self, - physical: bool, - var: str, - ) -> xr.Dataset: - """Get Copernicus Marine dataset for direct ingestion.""" - product_id = _select_product_id( - physical=physical, - schedule_start=self.expedition.schedule.space_time_region.time_range.start_time, - schedule_end=self.expedition.schedule.space_time_region.time_range.end_time, - variable=var if not physical else None, - ) + depth_min = self.limit_spec.get("depth_min") if self.limit_spec else None + depth_max = self.limit_spec.get("depth_max") if self.limit_spec else None return copernicusmarine.open_dataset( dataset_id=product_id, - dataset_part="default", minimum_longitude=self.expedition.schedule.space_time_region.spatial_range.minimum_longitude - - 3.0, + - latlon_buffer, maximum_longitude=self.expedition.schedule.space_time_region.spatial_range.maximum_longitude - + 3.0, + + latlon_buffer, minimum_latitude=self.expedition.schedule.space_time_region.spatial_range.minimum_latitude - - 3.0, + - latlon_buffer, maximum_latitude=self.expedition.schedule.space_time_region.spatial_range.maximum_latitude - + 3.0, - maximum_depth=1.0, - minimum_depth=1.0, + + latlon_buffer, variables=[var], start_datetime=self.expedition.schedule.space_time_region.time_range.start_time, end_datetime=self.expedition.schedule.space_time_region.time_range.end_time - + timedelta(days=21.0), + + timedelta(days=time_buffer), + minimum_depth=depth_min, + maximum_depth=depth_max, coordinates_selection_method="outside", ) diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index d205795b..9bd92353 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -96,6 +96,8 @@ def __init__(self, expedition, directory): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, + buffer_spec=None, + limit_spec=None, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 182d274c..4c286f3c 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -130,6 +130,8 @@ def __init__(self, expedition, directory): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, + buffer_spec=None, + limit_spec=None, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 06f6b71f..fee4e326 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -71,6 +71,15 @@ def __init__(self, expedition, directory): "T": f"{Drifter.name}_t.nc", } variables = {"U": "uo", "V": "vo", "T": "thetao"} + buffer_spec = { + "latlon": 6.0, # [degrees] + "time": 21.0, # [days] + } + limit_spec = { + "depth_min": 1.0, # [meters] + "depth_max": 1.0, # [meters] + } + super().__init__( Drifter.name, expedition, @@ -80,6 +89,8 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, + buffer_spec=buffer_spec, + limit_spec=limit_spec, ) def simulate(self, measurements, out_path) -> None: @@ -121,7 +132,7 @@ def simulate(self, measurements, out_path) -> None: chunks=[len(drifter_particleset), 100], ) - # get earliest between fieldset end time and provide end time + # get earliest between fieldset end time and prescribed end time fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) if ENDTIME is None: actual_endtime = fieldset_endtime diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index f6099869..32dcdd4f 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -72,6 +72,8 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, + buffer_spec=None, + limit_spec=None, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index fd88240d..2d6a7079 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -95,6 +95,8 @@ def __init__(self, expedition, directory): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, + buffer_spec=None, + limit_spec=None, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index aa310a97..0cb13955 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -132,7 +132,7 @@ def verify( if not ignore_missing_bathymetry: try: bathymetry_field = _get_bathy_data( - self.space_time_region + self.space_time_region, latlon_buffer=None ).bathymetry # via copernicusmarine except Exception as e: raise ScheduleError( @@ -406,9 +406,6 @@ def verify(self, expedition: Expedition) -> None: hasattr(self, config_attr) and inst_type not in instruments_in_expedition ): - print( - f"{inst_type.value} configuration provided but not in schedule. Removing config." - ) setattr(self, config_attr, None) # Check all scheduled instruments are configured for inst_type in instruments_in_expedition: diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 4ea17c4a..7e863034 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -412,14 +412,18 @@ def _start_end_in_product_timerange( ) -def _get_bathy_data(space_time_region) -> FieldSet: +def _get_bathy_data(space_time_region, latlon_buffer: float | None = None) -> FieldSet: """Bathymetry data 'streamed' directly from Copernicus Marine.""" ds_bathymetry = copernicusmarine.open_dataset( dataset_id="cmems_mod_glo_phy_my_0.083deg_static", - minimum_longitude=space_time_region.spatial_range.minimum_longitude, - maximum_longitude=space_time_region.spatial_range.maximum_longitude, - minimum_latitude=space_time_region.spatial_range.minimum_latitude, - maximum_latitude=space_time_region.spatial_range.maximum_latitude, + minimum_longitude=space_time_region.spatial_range.minimum_longitude + - (latlon_buffer if latlon_buffer is not None else 0), + maximum_longitude=space_time_region.spatial_range.maximum_longitude + + (latlon_buffer if latlon_buffer is not None else 0), + minimum_latitude=space_time_region.spatial_range.minimum_latitude + - (latlon_buffer if latlon_buffer is not None else 0), + maximum_latitude=space_time_region.spatial_range.maximum_latitude + + (latlon_buffer if latlon_buffer is not None else 0), variables=["deptho"], start_datetime=space_time_region.time_range.start_time, end_datetime=space_time_region.time_range.end_time, From 43afe97e19e9162c02da8225250b37a9c349559e Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 7 Nov 2025 13:18:24 +0100 Subject: [PATCH 57/97] remove _creds.py --- src/virtualship/cli/_creds.py | 108 ---------------------------------- 1 file changed, 108 deletions(-) delete mode 100644 src/virtualship/cli/_creds.py diff --git a/src/virtualship/cli/_creds.py b/src/virtualship/cli/_creds.py deleted file mode 100644 index d9169ec4..00000000 --- a/src/virtualship/cli/_creds.py +++ /dev/null @@ -1,108 +0,0 @@ -# - - -# TODO: TO DELETE?! - -from __future__ import annotations - -from pathlib import Path - -import click -import pydantic -import yaml - -from virtualship.errors import CredentialFileError - -CREDENTIALS_FILE = "credentials.yaml" - - -class Credentials(pydantic.BaseModel): - """Credentials to be used in `virtualship fetch` command.""" - - COPERNICUS_USERNAME: str - COPERNICUS_PASSWORD: str - - @classmethod - def from_yaml(cls, path: str | Path) -> Credentials: - """ - Load credentials from a yaml file. - - :param path: Path to the file to load from. - :returns Credentials: The credentials. - """ - with open(path) as file: - data = yaml.safe_load(file) - - if not isinstance(data, dict): - raise CredentialFileError("Credential file is of an invalid format.") - - return cls(**data) - - def dump(self) -> str: - """ - Dump credentials to a yaml string. - - :param creds: The credentials to dump. - :returns str: The yaml string. - """ - return yaml.safe_dump(self.model_dump()) - - def to_yaml(self, path: str | Path) -> None: - """ - Write credentials to a yaml file. - - :param path: Path to the file to write to. - """ - with open(path, "w") as file: - file.write(self.dump()) - - -def get_dummy_credentials_yaml() -> str: - return ( - Credentials( - COPERNICUS_USERNAME="my_username", COPERNICUS_PASSWORD="my_password" - ) - .dump() - .strip() - ) - - -def get_credentials_flow( - username: str | None, password: str | None, creds_path: Path -) -> tuple[str, str]: - """ - Execute flow of getting credentials for use in the `fetch` command. - - - If username and password are provided via CLI, use them (ignore the credentials file if exists). - - If username and password are not provided, try to load them from the credentials file. - - If no credentials are provided, print a message on how to make credentials file and prompt for credentials. - - :param username: The username provided via CLI. - :param password: The password provided via CLI. - :param creds_path: The path to the credentials file. - """ - if username and password: - if creds_path.exists(): - click.echo( - f"Credentials file exists at '{creds_path}', but username and password are already provided. Ignoring credentials file." - ) - return username, password - - try: - creds = Credentials.from_yaml(creds_path) - click.echo(f"Loaded credentials from '{creds_path}'.") - return creds.COPERNICUS_USERNAME, creds.COPERNICUS_PASSWORD - except FileNotFoundError: - msg = f"""Credentials not provided. Credentials can be obtained from https://data.marine.copernicus.eu/register. Either pass in via `--username` and `--password` arguments, or via config file at '{creds_path}'. Config file should be YAML along following format: -### {creds_path} - -{get_dummy_credentials_yaml().strip()} - -### - -Prompting for credentials instead... -""" - click.echo(msg) - username = click.prompt("username") - password = click.prompt("password", hide_input=True) - return username, password From e9a0c54e16525f4533aee918c189c37525763eca Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 7 Nov 2025 15:13:23 +0100 Subject: [PATCH 58/97] CTD_BGC fieldset bug fix --- src/virtualship/cli/_plan.py | 1 - src/virtualship/instruments/base.py | 83 ++++++++++++-------------- src/virtualship/instruments/ctd_bgc.py | 2 - 3 files changed, 38 insertions(+), 48 deletions(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 6aa6ff28..5ae46a82 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -1046,7 +1046,6 @@ def save_pressed(self) -> None: # verify schedule expedition_editor.expedition.schedule.verify( ship_speed_value, - bathy_data_dir=None, check_space_time_region=True, ignore_missing_bathymetry=True, ) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index c290f277..e597174b 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -4,13 +4,11 @@ from typing import TYPE_CHECKING import copernicusmarine -import numpy as np import xarray as xr from yaspin import yaspin from parcels import FieldSet from virtualship.utils import ( - COPERNICUSMARINE_BGC_VARIABLES, COPERNICUSMARINE_PHYS_VARIABLES, _get_bathy_data, _select_product_id, @@ -61,31 +59,16 @@ def __init__( def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" try: - datasets = [] - for var in self.variables.values(): - physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False - datasets.append(self._get_copernicus_ds(physical=physical, var=var)) - - # make sure time dims are matched if BGC variables are present (different monthly/daily resolutions can impact fieldset_endtime in simulate) - all_keys = set().union(*(ds.keys() for ds in datasets)) - if all_keys & set(COPERNICUSMARINE_BGC_VARIABLES): - datasets = self._align_temporal(datasets) - - ds_concat = xr.merge(datasets) # TODO: deal with WARNINGS? - - fieldset = FieldSet.from_xarray_dataset( - ds_concat, self.variables, self.dimensions, mesh="spherical" - ) - + fieldset = self._generate_fieldset() except Exception as e: raise FileNotFoundError( - f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'. " - f"Please check your credentials, network connection, and variable names. Original error: {e}" + f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'.Original error: {e}" ) from e # interpolation methods for var in (v for v in self.variables if v not in ("U", "V")): getattr(fieldset, var).interp_method = "linear_invdist_land_tracer" + # depth negative for g in fieldset.gridset.grids: g.negate_depth() @@ -109,18 +92,22 @@ def simulate(self, data_dir: Path, measurements: list, out_path: str | Path): def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - if not self.verbose_progress: - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: + TMP = False + if not TMP: + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate(measurements, out_path) + spinner.ok("✅\n") + else: + print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) - spinner.ok("✅\n") + print("\n") else: - print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) - print("\n") def _get_copernicus_ds( self, @@ -160,19 +147,25 @@ def _get_copernicus_ds( coordinates_selection_method="outside", ) - def _align_temporal(self, datasets: list[xr.Dataset]) -> list[xr.Dataset]: - """Align monthly and daily time dims of multiple datasets (by repeating monthly values daily).""" - reference_time = datasets[ - np.argmax(ds.time for ds in datasets) - ].time # daily timeseries - - datasets_aligned = [] - for ds in datasets: - if not np.array_equal(ds.time, reference_time): - # TODO: NEED TO CHOOSE BEST METHOD HERE - # ds = ds.resample(time="1D").ffill().reindex(time=reference_time) - # ds = ds.resample(time="1D").ffill() - ds = ds.reindex({"time": reference_time}, method="nearest") - datasets_aligned.append(ds) - - return datasets_aligned + def _generate_fieldset(self) -> FieldSet: + """ + Fieldset per variable then combine. + + Avoids issues when creating one FieldSet of ds's sourced from different Copernicus Marine product IDs, which is often the case for BGC variables. + + """ + fieldsets_list = [] + for key, var in self.variables.items(): + physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False + ds = self._get_copernicus_ds(physical=physical, var=var) + fieldset = FieldSet.from_xarray_dataset( + ds, {key: var}, self.dimensions, mesh="spherical" + ) + fieldsets_list.append(fieldset) + base_fieldset = fieldsets_list[0] + if len(fieldsets_list) > 1: + for fs, key in zip( + fieldsets_list[1:], list(self.variables.keys())[1:], strict=True + ): + base_fieldset.add_field(getattr(fs, key)) + return base_fieldset diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 4c286f3c..33c17d99 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -205,8 +205,6 @@ def simulate(self, measurements, out_path) -> None: # define output file for the simulation out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=OUTPUT_DT) - breakpoint() - # execute simulation ctd_bgc_particleset.execute( [ From 995a6e57635570fb1b1faf8cb59ca0c2a4360fb9 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 10 Nov 2025 09:52:40 +0100 Subject: [PATCH 59/97] fixing bugs associated with BGC data access --- src/virtualship/cli/_run.py | 11 ++-- src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/instruments/base.py | 62 ++++++++++++----------- 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index df8a24fd..29bf0f7b 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -3,6 +3,7 @@ import logging import os import shutil +import time from pathlib import Path import pyproj @@ -39,12 +40,9 @@ def _run(expedition_dir: str | Path) -> None: :param expedition_dir: The base directory for the expedition. """ - # ################################# TEMPORARY TIMER: START ################################# - import time - + # start timing start_time = time.time() print("[TIMER] Expedition started...") - # ################################# TEMPORARY TIMER: START ################################# print("\n╔═════════════════════════════════════════════════╗") print("║ VIRTUALSHIP EXPEDITION STATUS ║") @@ -139,11 +137,10 @@ def _run(expedition_dir: str | Path) -> None: ) print("\n------------- END -------------\n") - ################################# TEMPORARY TIMER: END ################################# + # end timing end_time = time.time() elapsed = end_time - start_time - print(f"[TIMER] Expedition completed in {elapsed:.2f} seconds.") - ################################# TEMPORARY TIMER: END ################################# + print(f"[TIMER] Expedition completed in {elapsed / 60.0:.2f} minutes.") def _load_checkpoint(expedition_dir: Path) -> Checkpoint | None: diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 12ee8945..7935ca55 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -147,7 +147,7 @@ def __init__(self, expedition, directory): } variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} buffer_spec = { - "latlon": 6.0, # [degrees] + "latlon": 3.0, # [degrees] "time": 21.0, # [days] } diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index e597174b..06d8b8c0 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,4 +1,5 @@ import abc +from collections import OrderedDict from datetime import timedelta from pathlib import Path from typing import TYPE_CHECKING @@ -8,6 +9,7 @@ from yaspin import yaspin from parcels import FieldSet +from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, _get_bathy_data, @@ -43,7 +45,8 @@ def __init__( self.expedition = expedition self.directory = directory self.filenames = filenames - self.variables = variables + + self.variables = OrderedDict(variables) self.dimensions = { "lon": "longitude", "lat": "latitude", @@ -61,8 +64,8 @@ def load_input_data(self) -> FieldSet: try: fieldset = self._generate_fieldset() except Exception as e: - raise FileNotFoundError( - f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'.Original error: {e}" + raise CopernicusCatalogueError( + f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'. Original error: {e}" ) from e # interpolation methods @@ -92,22 +95,18 @@ def simulate(self, data_dir: Path, measurements: list, out_path: str | Path): def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - TMP = False - if not TMP: - if not self.verbose_progress: - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: - self.simulate(measurements, out_path) - spinner.ok("✅\n") - else: - print(f"Simulating {self.name} measurements... ") + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: self.simulate(measurements, out_path) - print("\n") + spinner.ok("✅\n") else: + print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) + print("\n") def _get_copernicus_ds( self, @@ -122,11 +121,10 @@ def _get_copernicus_ds( variable=var if not physical else None, ) - latlon_buffer = self.buffer_spec.get("latlon") if self.buffer_spec else 0.0 - time_buffer = self.buffer_spec.get("time") if self.buffer_spec else 0.0 - - depth_min = self.limit_spec.get("depth_min") if self.limit_spec else None - depth_max = self.limit_spec.get("depth_max") if self.limit_spec else None + latlon_buffer = self._get_spec_value("buffer", "latlon", 0.0) + time_buffer = self._get_spec_value("buffer", "time", 0.0) + depth_min = self._get_spec_value("limit", "depth_min", None) + depth_max = self._get_spec_value("limit", "depth_max", None) return copernicusmarine.open_dataset( dataset_id=product_id, @@ -151,21 +149,27 @@ def _generate_fieldset(self) -> FieldSet: """ Fieldset per variable then combine. - Avoids issues when creating one FieldSet of ds's sourced from different Copernicus Marine product IDs, which is often the case for BGC variables. - + Avoids issues when creating directly one FieldSet of ds's sourced from different Copernicus Marine product IDs, which is often the case for BGC variables. """ fieldsets_list = [] - for key, var in self.variables.items(): + keys = list(self.variables.keys()) + for key in keys: + var = self.variables[key] physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False ds = self._get_copernicus_ds(physical=physical, var=var) - fieldset = FieldSet.from_xarray_dataset( + fs = FieldSet.from_xarray_dataset( ds, {key: var}, self.dimensions, mesh="spherical" ) - fieldsets_list.append(fieldset) + fieldsets_list.append(fs) + base_fieldset = fieldsets_list[0] if len(fieldsets_list) > 1: - for fs, key in zip( - fieldsets_list[1:], list(self.variables.keys())[1:], strict=True - ): + for fs, key in zip(fieldsets_list[1:], keys[1:], strict=True): base_fieldset.add_field(getattr(fs, key)) + return base_fieldset + + def _get_spec_value(self, spec_type: str, key: str, default=None): + """Helper to extract a value from buffer_spec or limit_spec.""" + spec = self.buffer_spec if spec_type == "buffer" else self.limit_spec + return spec.get(key) if spec and spec.get(key) is not None else default From ec3329672ddcf9fad9d33fa21ab714651024aa54 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 10 Nov 2025 10:18:41 +0100 Subject: [PATCH 60/97] update dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 9862463b..7f9a2108 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "copernicusmarine >= 2.2.2", "yaspin", "textual", + "openpyxl", ] [project.urls] From c1321ba25d5507ffc2ee75d49ede93b51e2be5d6 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 10 Nov 2025 10:32:53 +0100 Subject: [PATCH 61/97] logic for handling copernicus credentials --- src/virtualship/cli/_run.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index 29bf0f7b..bf98f562 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -6,6 +6,7 @@ import time from pathlib import Path +import copernicusmarine import pyproj from virtualship.expedition.simulate_schedule import ( @@ -48,6 +49,21 @@ def _run(expedition_dir: str | Path) -> None: print("║ VIRTUALSHIP EXPEDITION STATUS ║") print("╚═════════════════════════════════════════════════╝") + COPERNICUS_CREDS_FILE = os.path.expandvars( + "$HOME/.copernicusmarine/.copernicusmarine-credentials" + ) + + if ( + os.path.isfile(COPERNICUS_CREDS_FILE) + and os.path.getsize(COPERNICUS_CREDS_FILE) > 0 + ): + pass + else: + print( + "\nPlease enter your log in details for the Copernicus Marine Service (only necessary the first time you run VirtualShip). \n\nIf you have not registered yet, please do so at https://marine.copernicus.eu/.\n" + ) + copernicusmarine.login() + if isinstance(expedition_dir, str): expedition_dir = Path(expedition_dir) From f386f42aad476950f8f0c863c202d359dc4be573 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 10 Nov 2025 15:10:20 +0100 Subject: [PATCH 62/97] add support for taking local pre-downloaded data with --from-data optional flag in virtualship run --- src/virtualship/cli/_plan.py | 2 +- src/virtualship/cli/_run.py | 30 ++--- src/virtualship/cli/commands.py | 15 ++- src/virtualship/instruments/adcp.py | 3 +- src/virtualship/instruments/argo_float.py | 3 +- src/virtualship/instruments/base.py | 116 ++++++++++++++---- src/virtualship/instruments/ctd.py | 3 +- src/virtualship/instruments/ctd_bgc.py | 3 +- src/virtualship/instruments/drifter.py | 3 +- .../instruments/ship_underwater_st.py | 3 +- src/virtualship/instruments/xbt.py | 3 +- src/virtualship/models/expedition.py | 9 +- src/virtualship/utils.py | 66 +++++++--- 13 files changed, 190 insertions(+), 69 deletions(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 5ae46a82..a164cba3 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -1047,7 +1047,7 @@ def save_pressed(self) -> None: expedition_editor.expedition.schedule.verify( ship_speed_value, check_space_time_region=True, - ignore_missing_bathymetry=True, + ignore_land_test=True, ) expedition_saved = expedition_editor.save_changes() diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index bf98f562..fac09142 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -35,7 +35,7 @@ logging.getLogger("copernicusmarine").setLevel("ERROR") -def _run(expedition_dir: str | Path) -> None: +def _run(expedition_dir: str | Path, from_data: Path | None = None) -> None: """ Perform an expedition, providing terminal feedback and file output. @@ -49,20 +49,21 @@ def _run(expedition_dir: str | Path) -> None: print("║ VIRTUALSHIP EXPEDITION STATUS ║") print("╚═════════════════════════════════════════════════╝") - COPERNICUS_CREDS_FILE = os.path.expandvars( - "$HOME/.copernicusmarine/.copernicusmarine-credentials" - ) - - if ( - os.path.isfile(COPERNICUS_CREDS_FILE) - and os.path.getsize(COPERNICUS_CREDS_FILE) > 0 - ): - pass - else: - print( - "\nPlease enter your log in details for the Copernicus Marine Service (only necessary the first time you run VirtualShip). \n\nIf you have not registered yet, please do so at https://marine.copernicus.eu/.\n" + if from_data is None: + COPERNICUS_CREDS_FILE = os.path.expandvars( + "$HOME/.copernicusmarine/.copernicusmarine-credentials" ) - copernicusmarine.login() + + if ( + os.path.isfile(COPERNICUS_CREDS_FILE) + and os.path.getsize(COPERNICUS_CREDS_FILE) > 0 + ): + pass + else: + print( + "\nPlease enter your log in details for the Copernicus Marine Service (only necessary the first time you run VirtualShip). \n\nIf you have not registered yet, please do so at https://marine.copernicus.eu/.\n" + ) + copernicusmarine.login() if isinstance(expedition_dir, str): expedition_dir = Path(expedition_dir) @@ -136,6 +137,7 @@ def _run(expedition_dir: str | Path) -> None: instrument = instrument_class( expedition=expedition, directory=expedition_dir, + from_data=Path(from_data) if from_data is not None else None, ) # execute simulation diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index 2b0ce8e9..f2f9f272 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -6,6 +6,8 @@ from virtualship.cli._plan import _plan from virtualship.cli._run import _run from virtualship.utils import ( + COPERNICUSMARINE_BGC_VARIABLES, + COPERNICUSMARINE_PHYS_VARIABLES, EXPEDITION, mfp_to_yaml, ) @@ -81,6 +83,15 @@ def plan(path): "path", type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True), ) -def run(path): +@click.option( + "--from-data", + type=str, + default=None, + help="Use pre-downloaded data, saved to disk, for expedition, instead of streaming directly via Copernicus Marine" + "Assumes all data is stored in prescribed directory, and all variables (as listed below) are present." + f"Required variables are: {set(COPERNICUSMARINE_PHYS_VARIABLES + COPERNICUSMARINE_BGC_VARIABLES)}" + "Assumes that variable names at least contain the standard Copernicus Marine variable name as a substring.", +) +def run(path, from_data): """Execute the expedition simulations.""" - _run(Path(path)) + _run(Path(path), from_data) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index dd675b7e..8dca31a3 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -54,7 +54,7 @@ def _sample_velocity(particle, fieldset, time): class ADCPInstrument(Instrument): """ADCP instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_data): """Initialize ADCPInstrument.""" filenames = { "U": f"{ADCP.name}_uv.nc", @@ -73,6 +73,7 @@ def __init__(self, expedition, directory): verbose_progress=False, buffer_spec=None, limit_spec=None, + from_data=from_data, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 7935ca55..8ba794ba 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -137,7 +137,7 @@ def _check_error(particle, fieldset, time): class ArgoFloatInstrument(Instrument): """ArgoFloat instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_data): """Initialize ArgoFloatInstrument.""" filenames = { "U": f"{ArgoFloat.name}_uv.nc", @@ -162,6 +162,7 @@ def __init__(self, expedition, directory): verbose_progress=True, buffer_spec=buffer_spec, limit_spec=None, + from_data=from_data, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 06d8b8c0..7941bf18 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -12,6 +12,7 @@ from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, + _find_nc_file_with_variable, _get_bathy_data, _select_product_id, ship_spinner, @@ -24,9 +25,6 @@ class Instrument(abc.ABC): """Base class for instruments and their simulation.""" - #! TODO List: - # TODO: how is this handling credentials?! Seems to work already, are these set up from my previous instances of using copernicusmarine? Therefore users will only have to do it once too? - def __init__( self, name: str, @@ -37,6 +35,7 @@ def __init__( add_bathymetry: bool, allow_time_extrapolation: bool, verbose_progress: bool, + from_data: Path | None, buffer_spec: dict | None = None, limit_spec: dict | None = None, ): @@ -45,6 +44,7 @@ def __init__( self.expedition = expedition self.directory = directory self.filenames = filenames + self.from_data = from_data self.variables = OrderedDict(variables) self.dimensions = { @@ -65,7 +65,7 @@ def load_input_data(self) -> FieldSet: fieldset = self._generate_fieldset() except Exception as e: raise CopernicusCatalogueError( - f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'. Original error: {e}" + f"Failed to load input data directly from Copernicus Marine (or local data) for instrument '{self.name}'. Original error: {e}" ) from e # interpolation methods @@ -83,6 +83,7 @@ def load_input_data(self) -> FieldSet: latlon_buffer=self.buffer_spec.get("latlon") if self.buffer_spec else None, + from_data=self.from_data, ).bathymetry bathymetry_field.data = -bathymetry_field.data fieldset.add_field(bathymetry_field) @@ -90,23 +91,36 @@ def load_input_data(self) -> FieldSet: return fieldset @abc.abstractmethod - def simulate(self, data_dir: Path, measurements: list, out_path: str | Path): + def simulate( + self, + data_dir: Path, + measurements: list, + out_path: str | Path, + ) -> None: """Simulate instrument measurements.""" def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - if not self.verbose_progress: - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: + # + TMP = True # temporary spinner implementation + # + if TMP: + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate(measurements, out_path) + spinner.ok("✅\n") + else: + print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) - spinner.ok("✅\n") + print("\n") + + # else: - print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) - print("\n") def _get_copernicus_ds( self, @@ -145,27 +159,81 @@ def _get_copernicus_ds( coordinates_selection_method="outside", ) + def _load_local_ds(self, filename) -> xr.Dataset: + """ + Load local dataset from specified data directory. + + Sliced according to expedition.schedule.space_time_region andbuffer and limit specs. + """ + ds = xr.open_dataset(self.from_data.joinpath(filename)) + + coord_rename = {} + if "lat" in ds.coords: + coord_rename["lat"] = "latitude" + if "lon" in ds.coords: + coord_rename["lon"] = "longitude" + if coord_rename: + ds = ds.rename(coord_rename) + + min_lon = ( + self.expedition.schedule.space_time_region.spatial_range.minimum_longitude + - self._get_spec_value( + "buffer", "latlon", 3.0 + ) # always add min 3 deg buffer for local data to avoid edge issues with ds.sel() + ) + max_lon = ( + self.expedition.schedule.space_time_region.spatial_range.maximum_longitude + + self._get_spec_value("buffer", "latlon", 3.0) + ) + min_lat = ( + self.expedition.schedule.space_time_region.spatial_range.minimum_latitude + - self._get_spec_value("buffer", "latlon", 3.0) + ) + max_lat = ( + self.expedition.schedule.space_time_region.spatial_range.maximum_latitude + + self._get_spec_value("buffer", "latlon", 3.0) + ) + min_depth = self._get_spec_value("limit", "depth_min", None) + max_depth = self._get_spec_value("limit", "depth_max", None) + + return ds.sel( + latitude=slice(min_lat, max_lat), + longitude=slice(min_lon, max_lon), + depth=slice(min_depth, max_depth) + if min_depth is not None and max_depth is not None + else slice(None), + ) + def _generate_fieldset(self) -> FieldSet: """ - Fieldset per variable then combine. + Create and combine FieldSets for each variable, supporting both local and Copernicus Marine data sources. - Avoids issues when creating directly one FieldSet of ds's sourced from different Copernicus Marine product IDs, which is often the case for BGC variables. + Avoids issues when using copernicusmarine and creating directly one FieldSet of ds's sourced from different Copernicus Marine product IDs, which is often the case for BGC variables. """ fieldsets_list = [] keys = list(self.variables.keys()) + for key in keys: var = self.variables[key] - physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False - ds = self._get_copernicus_ds(physical=physical, var=var) - fs = FieldSet.from_xarray_dataset( - ds, {key: var}, self.dimensions, mesh="spherical" - ) + if self.from_data is not None: # load from local data + filename, full_var_name = _find_nc_file_with_variable( + self.from_data, var + ) + ds = self._load_local_ds(filename) + fs = FieldSet.from_xarray_dataset( + ds, {key: full_var_name}, self.dimensions, mesh="spherical" + ) + else: # steam via Copernicus Marine + physical = var in COPERNICUSMARINE_PHYS_VARIABLES + ds = self._get_copernicus_ds(physical=physical, var=var) + fs = FieldSet.from_xarray_dataset( + ds, {key: var}, self.dimensions, mesh="spherical" + ) fieldsets_list.append(fs) base_fieldset = fieldsets_list[0] - if len(fieldsets_list) > 1: - for fs, key in zip(fieldsets_list[1:], keys[1:], strict=True): - base_fieldset.add_field(getattr(fs, key)) + for fs, key in zip(fieldsets_list[1:], keys[1:], strict=False): + base_fieldset.add_field(getattr(fs, key)) return base_fieldset diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 9bd92353..04fb24ed 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -79,7 +79,7 @@ def _ctd_cast(particle, fieldset, time): class CTDInstrument(Instrument): """CTD instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_data): """Initialize CTDInstrument.""" filenames = { "S": f"{CTD.name}_s.nc", @@ -98,6 +98,7 @@ def __init__(self, expedition, directory): verbose_progress=False, buffer_spec=None, limit_spec=None, + from_data=from_data, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 33c17d99..59dc4bc2 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -101,7 +101,7 @@ def _ctd_bgc_cast(particle, fieldset, time): class CTD_BGCInstrument(Instrument): """CTD_BGC instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_data): """Initialize CTD_BGCInstrument.""" filenames = { "o2": f"{CTD_BGC.name}_o2.nc", @@ -132,6 +132,7 @@ def __init__(self, expedition, directory): verbose_progress=False, buffer_spec=None, limit_spec=None, + from_data=from_data, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index fee4e326..8ad0fe19 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -63,7 +63,7 @@ def _check_lifetime(particle, fieldset, time): class DrifterInstrument(Instrument): """Drifter instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_data): """Initialize DrifterInstrument.""" filenames = { "U": f"{Drifter.name}_uv.nc", @@ -91,6 +91,7 @@ def __init__(self, expedition, directory): verbose_progress=True, buffer_spec=buffer_spec, limit_spec=limit_spec, + from_data=from_data, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 32dcdd4f..a28ae12d 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -55,7 +55,7 @@ def _sample_temperature(particle, fieldset, time): class Underwater_STInstrument(Instrument): """Underwater_ST instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_data): """Initialize Underwater_STInstrument.""" filenames = { "S": f"{Underwater_ST.name}_s.nc", @@ -74,6 +74,7 @@ def __init__(self, expedition, directory): verbose_progress=False, buffer_spec=None, limit_spec=None, + from_data=from_data, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 2d6a7079..5972efff 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -77,7 +77,7 @@ def _xbt_cast(particle, fieldset, time): class XBTInstrument(Instrument): """XBT instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_data): """Initialize XBTInstrument.""" filenames = { "U": f"{XBT.name}_uv.nc", @@ -97,6 +97,7 @@ def __init__(self, expedition, directory): verbose_progress=False, buffer_spec=None, limit_spec=None, + from_data=from_data, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 0cb13955..0b82e0af 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -87,7 +87,7 @@ class Schedule(pydantic.BaseModel): def verify( self, ship_speed: float, - ignore_missing_bathymetry: bool = False, + ignore_land_test: bool = False, *, check_space_time_region: bool = False, ) -> None: @@ -129,11 +129,12 @@ def verify( # check if all waypoints are in water using bathymetry data # TODO: write test that checks that will flag when waypoint is on land!! [add to existing suite of fail .verify() tests in test_expedition.py] land_waypoints = [] - if not ignore_missing_bathymetry: + if not ignore_land_test: try: bathymetry_field = _get_bathy_data( - self.space_time_region, latlon_buffer=None - ).bathymetry # via copernicusmarine + self.space_time_region, + latlon_buffer=None, + ).bathymetry except Exception as e: raise ScheduleError( f"Problem loading bathymetry data (used to verify waypoints are in water) directly via copernicusmarine. \n\n original message: {e}" diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 7e863034..9f1a08e8 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -10,6 +10,7 @@ import copernicusmarine import numpy as np +import xarray as xr from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError @@ -412,23 +413,41 @@ def _start_end_in_product_timerange( ) -def _get_bathy_data(space_time_region, latlon_buffer: float | None = None) -> FieldSet: - """Bathymetry data 'streamed' directly from Copernicus Marine.""" - ds_bathymetry = copernicusmarine.open_dataset( - dataset_id="cmems_mod_glo_phy_my_0.083deg_static", - minimum_longitude=space_time_region.spatial_range.minimum_longitude - - (latlon_buffer if latlon_buffer is not None else 0), - maximum_longitude=space_time_region.spatial_range.maximum_longitude - + (latlon_buffer if latlon_buffer is not None else 0), - minimum_latitude=space_time_region.spatial_range.minimum_latitude - - (latlon_buffer if latlon_buffer is not None else 0), - maximum_latitude=space_time_region.spatial_range.maximum_latitude - + (latlon_buffer if latlon_buffer is not None else 0), - variables=["deptho"], - start_datetime=space_time_region.time_range.start_time, - end_datetime=space_time_region.time_range.end_time, - coordinates_selection_method="outside", - ) +def _get_bathy_data( + space_time_region, latlon_buffer: float | None = None, from_data: Path | None = None +) -> FieldSet: + """Bathymetry data from local or 'streamed' directly from Copernicus Marine.""" + if from_data is not None: # load from local data + var = "deptho" + try: + filename, _ = _find_nc_file_with_variable(from_data, var) + except Exception as e: + raise RuntimeError( + f"Could not find bathymetry variable '{var}' in provided data directory '{from_data}'." + ) from e + ds_bathymetry = xr.open_dataset(from_data.joinpath(filename)) + bathymetry_variables = {"bathymetry": "deptho"} + bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} + return FieldSet.from_xarray_dataset( + ds_bathymetry, bathymetry_variables, bathymetry_dimensions + ) + + else: # stream via Copernicus Marine + ds_bathymetry = copernicusmarine.open_dataset( + dataset_id="cmems_mod_glo_phy_my_0.083deg_static", + minimum_longitude=space_time_region.spatial_range.minimum_longitude + - (latlon_buffer if latlon_buffer is not None else 0), + maximum_longitude=space_time_region.spatial_range.maximum_longitude + + (latlon_buffer if latlon_buffer is not None else 0), + minimum_latitude=space_time_region.spatial_range.minimum_latitude + - (latlon_buffer if latlon_buffer is not None else 0), + maximum_latitude=space_time_region.spatial_range.maximum_latitude + + (latlon_buffer if latlon_buffer is not None else 0), + variables=["deptho"], + start_datetime=space_time_region.time_range.start_time, + end_datetime=space_time_region.time_range.end_time, + coordinates_selection_method="outside", + ) bathymetry_variables = {"bathymetry": "deptho"} bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} return FieldSet.from_xarray_dataset( @@ -456,3 +475,16 @@ def expedition_cost(schedule_results: ScheduleOk, time_past: timedelta) -> float cost = ship_cost + argo_cost + drifter_cost return cost + + +def _find_nc_file_with_variable(data_dir: Path, var: str) -> str | None: + """Search for a .nc file in the given directory containing the specified variable.""" + for nc_file in data_dir.glob("*.nc"): + try: + with xr.open_dataset(nc_file, chunks={}) as ds: + matched_vars = [v for v in ds.variables if var in v] + if matched_vars: + return nc_file.name, matched_vars[0] + except Exception: + continue + return None From 7ba83fc30c23857361ca4a4a13f3ee70dcec914f Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 10 Nov 2025 15:45:09 +0100 Subject: [PATCH 63/97] update drifter to be at -1m depth, to avoid out of bounds at surface --- src/virtualship/instruments/base.py | 7 +------ src/virtualship/static/expedition.yaml | 2 +- tests/expedition/expedition_dir/expedition.yaml | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 7941bf18..5c764cb3 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -163,7 +163,7 @@ def _load_local_ds(self, filename) -> xr.Dataset: """ Load local dataset from specified data directory. - Sliced according to expedition.schedule.space_time_region andbuffer and limit specs. + Sliced according to expedition.schedule.space_time_region and buffer specs. """ ds = xr.open_dataset(self.from_data.joinpath(filename)) @@ -193,15 +193,10 @@ def _load_local_ds(self, filename) -> xr.Dataset: self.expedition.schedule.space_time_region.spatial_range.maximum_latitude + self._get_spec_value("buffer", "latlon", 3.0) ) - min_depth = self._get_spec_value("limit", "depth_min", None) - max_depth = self._get_spec_value("limit", "depth_max", None) return ds.sel( latitude=slice(min_lat, max_lat), longitude=slice(min_lon, max_lon), - depth=slice(min_depth, max_depth) - if min_depth is not None and max_depth is not None - else slice(None), ) def _generate_fieldset(self) -> FieldSet: diff --git a/src/virtualship/static/expedition.yaml b/src/virtualship/static/expedition.yaml index 1a9e3922..256bee87 100644 --- a/src/virtualship/static/expedition.yaml +++ b/src/virtualship/static/expedition.yaml @@ -62,7 +62,7 @@ instruments_config: min_depth_meter: -11.0 stationkeeping_time_minutes: 20.0 drifter_config: - depth_meter: 0.0 + depth_meter: -1.0 lifetime_minutes: 60480.0 xbt_config: max_depth_meter: -285.0 diff --git a/tests/expedition/expedition_dir/expedition.yaml b/tests/expedition/expedition_dir/expedition.yaml index 9468028f..0ed9e5f4 100644 --- a/tests/expedition/expedition_dir/expedition.yaml +++ b/tests/expedition/expedition_dir/expedition.yaml @@ -38,7 +38,7 @@ instruments_config: min_depth_meter: -11.0 stationkeeping_time_minutes: 20.0 drifter_config: - depth_meter: 0.0 + depth_meter: -1.0 lifetime_minutes: 40320.0 ship_underwater_st_config: period_minutes: 5.0 From f13c8363e5c2ac760716c02b7f919316d1d71c58 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 10 Nov 2025 15:45:19 +0100 Subject: [PATCH 64/97] tidy up --- src/virtualship/cli/_run.py | 2 ++ src/virtualship/cli/commands.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index fac09142..353f6a1f 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -50,6 +50,8 @@ def _run(expedition_dir: str | Path, from_data: Path | None = None) -> None: print("╚═════════════════════════════════════════════════╝") if from_data is None: + # TODO: caution, if collaborative environments, will this mean everyone uses the same credentials file? + # TODO: may need to think about how to deal with this if using collaborative environments AND streaming data via copernicusmarine COPERNICUS_CREDS_FILE = os.path.expandvars( "$HOME/.copernicusmarine/.copernicusmarine-credentials" ) diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index f2f9f272..3e485497 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -2,13 +2,13 @@ import click -from virtualship import utils from virtualship.cli._plan import _plan from virtualship.cli._run import _run from virtualship.utils import ( COPERNICUSMARINE_BGC_VARIABLES, COPERNICUSMARINE_PHYS_VARIABLES, EXPEDITION, + get_example_expedition, mfp_to_yaml, ) @@ -58,7 +58,7 @@ def init(path, from_mfp): ) else: # Create a default example expedition YAML - expedition.write_text(utils.get_example_expedition()) + expedition.write_text(get_example_expedition()) click.echo(f"Created '{expedition.name}' at {path}.") @@ -90,7 +90,8 @@ def plan(path): help="Use pre-downloaded data, saved to disk, for expedition, instead of streaming directly via Copernicus Marine" "Assumes all data is stored in prescribed directory, and all variables (as listed below) are present." f"Required variables are: {set(COPERNICUSMARINE_PHYS_VARIABLES + COPERNICUSMARINE_BGC_VARIABLES)}" - "Assumes that variable names at least contain the standard Copernicus Marine variable name as a substring.", + "Assumes that variable names at least contain the standard Copernicus Marine variable name as a substring." + "Will also take the first file found containing the variable name substring. CAUTION if multiple files contain the same variable name substring.", ) def run(path, from_data): """Execute the expedition simulations.""" From 4f533c52bfbdcea23a174789873ec730ec374a72 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Nov 2025 09:31:43 +0100 Subject: [PATCH 65/97] bug fixes for unnecessary copernicusmarine call when using pre-downloaded data --- src/virtualship/cli/_run.py | 8 ++++++-- src/virtualship/models/expedition.py | 3 +++ src/virtualship/utils.py | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index 353f6a1f..cf83da7d 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -63,7 +63,8 @@ def _run(expedition_dir: str | Path, from_data: Path | None = None) -> None: pass else: print( - "\nPlease enter your log in details for the Copernicus Marine Service (only necessary the first time you run VirtualShip). \n\nIf you have not registered yet, please do so at https://marine.copernicus.eu/.\n" + "\nPlease enter your log in details for the Copernicus Marine Service (only necessary the first time you run VirtualShip). \n\nIf you have not registered yet, please do so at https://marine.copernicus.eu/.\n\n" + "If you did not expect to see this message, and intended to use pre-downloaded data instead of streaming via Copernicus Marine, please use the '--from-data' option to specify the path to the data.\n" ) copernicusmarine.login() @@ -85,7 +86,10 @@ def _run(expedition_dir: str | Path, from_data: Path | None = None) -> None: print("\n---- WAYPOINT VERIFICATION ----") - expedition.schedule.verify(expedition.ship_config.ship_speed_knots) + expedition.schedule.verify( + expedition.ship_config.ship_speed_knots, + from_data=Path(from_data) if from_data else None, + ) # simulate the schedule schedule_results = simulate_schedule( diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 0b82e0af..96e591b3 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -2,6 +2,7 @@ import itertools from datetime import datetime, timedelta +from pathlib import Path from typing import TYPE_CHECKING import pydantic @@ -90,6 +91,7 @@ def verify( ignore_land_test: bool = False, *, check_space_time_region: bool = False, + from_data: Path | None = None, ) -> None: """ Verify the feasibility and correctness of the schedule's waypoints. @@ -134,6 +136,7 @@ def verify( bathymetry_field = _get_bathy_data( self.space_time_region, latlon_buffer=None, + from_data=from_data, ).bathymetry except Exception as e: raise ScheduleError( diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 9f1a08e8..d44c528f 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -448,8 +448,9 @@ def _get_bathy_data( end_datetime=space_time_region.time_range.end_time, coordinates_selection_method="outside", ) - bathymetry_variables = {"bathymetry": "deptho"} - bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} + bathymetry_variables = {"bathymetry": "deptho"} + bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} + return FieldSet.from_xarray_dataset( ds_bathymetry, bathymetry_variables, bathymetry_dimensions ) From d288ead03fdafded2b235ee74c73a30b4c15a6bf Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Nov 2025 09:51:48 +0100 Subject: [PATCH 66/97] remove redundant tests --- tests/cli/test_cli.py | 23 +-- tests/cli/test_creds.py | 66 --------- tests/cli/test_fetch.py | 139 ------------------ .../test_do_expedition.py => cli/test_run.py} | 6 +- 4 files changed, 4 insertions(+), 230 deletions(-) delete mode 100644 tests/cli/test_creds.py delete mode 100644 tests/cli/test_fetch.py rename tests/{expedition/test_do_expedition.py => cli/test_run.py} (54%) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 1b2c6e53..a0d90d77 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -5,7 +5,7 @@ import xarray as xr from click.testing import CliRunner -from virtualship.cli.commands import fetch, init +from virtualship.cli.commands import init from virtualship.utils import EXPEDITION @@ -65,24 +65,3 @@ def test_init_existing_expedition(): with pytest.raises(FileExistsError): result = runner.invoke(init, ["."]) raise result.exception - - -@pytest.mark.parametrize( - "fetch_args", - [ - [".", "--username", "test"], - [".", "--password", "test"], - ], -) -@pytest.mark.usefixtures("copernicus_no_download") -def test_fetch_both_creds_via_cli(runner, fetch_args): - result = runner.invoke(fetch, fetch_args) - assert result.exit_code == 1 - assert "Both username and password" in result.exc_info[1].args[0] - - -@pytest.mark.usefixtures("copernicus_no_download") -def test_fetch(runner): - """Test the fetch command, but mock the downloads (and metadata interrogation).""" - result = runner.invoke(fetch, [".", "--username", "test", "--password", "test"]) - assert result.exit_code == 0 diff --git a/tests/cli/test_creds.py b/tests/cli/test_creds.py deleted file mode 100644 index 17ef2023..00000000 --- a/tests/cli/test_creds.py +++ /dev/null @@ -1,66 +0,0 @@ -import pydantic -import pytest - -from virtualship.cli._creds import CredentialFileError, Credentials - - -def test_load_credentials(tmp_file): - tmp_file.write_text( - """ - COPERNICUS_USERNAME: test_user - COPERNICUS_PASSWORD: test_password - """ - ) - - creds = Credentials.from_yaml(tmp_file) - assert creds.COPERNICUS_USERNAME == "test_user" - assert creds.COPERNICUS_PASSWORD == "test_password" - - -# parameterize with the contents of the file -@pytest.mark.parametrize( - "contents", - [ - pytest.param( - """ - INVALID_KEY: some_value - """, - id="invalid-key", - ), - pytest.param( - """ - # number not allowed, should be string (or quoted number) - USERNAME: 123 - """, - id="number-not-allowed", - ), - ], -) -def test_invalid_credentials(tmp_file, contents): - tmp_file.write_text(contents) - - with pytest.raises(pydantic.ValidationError): - Credentials.from_yaml(tmp_file) - - -def test_credentials_invalid_format(tmp_file): - tmp_file.write_text( - """ - INVALID_FORMAT_BUT_VALID_YAML - """ - ) - - with pytest.raises(CredentialFileError): - Credentials.from_yaml(tmp_file) - - -def test_rt_credentials(tmp_file): - """Test round-trip for credentials using Credentials.from_yaml() and Credentials.dump().""" - creds = Credentials( - COPERNICUS_USERNAME="test_user", COPERNICUS_PASSWORD="test_password" - ) - - creds.to_yaml(tmp_file) - creds_loaded = Credentials.from_yaml(tmp_file) - - assert creds == creds_loaded diff --git a/tests/cli/test_fetch.py b/tests/cli/test_fetch.py deleted file mode 100644 index 4c041dbe..00000000 --- a/tests/cli/test_fetch.py +++ /dev/null @@ -1,139 +0,0 @@ -from pathlib import Path - -import numpy as np -import pytest -import xarray as xr -from pydantic import BaseModel - -from virtualship.cli._fetch import ( - DOWNLOAD_METADATA, - DownloadMetadata, - IncompleteDownloadError, - _fetch, - assert_complete_download, - complete_download, - create_hash, - filename_to_hash, - get_existing_download, - hash_model, - hash_to_filename, -) -from virtualship.models import Expedition -from virtualship.utils import EXPEDITION, get_example_expedition - - -@pytest.fixture -def copernicus_no_download(monkeypatch): - """Mock the copernicusmarine `subset` and `open_dataset` functions, approximating the reanalysis products.""" - - # mock for copernicusmarine.subset - def fake_download(output_filename, output_directory, **_): - Path(output_directory).joinpath(output_filename).touch() - - def fake_open_dataset(*args, **kwargs): - return xr.Dataset( - coords={ - "time": ( - "time", - [ - np.datetime64("2022-01-01"), - np.datetime64("2025-01-01"), - ], # mock up rough renanalysis period - ) - } - ) - - monkeypatch.setattr("virtualship.cli._fetch.copernicusmarine.subset", fake_download) - monkeypatch.setattr( - "virtualship.cli._fetch.copernicusmarine.open_dataset", fake_open_dataset - ) - yield - - -@pytest.fixture -def expedition(tmpdir): - out_path = tmpdir.join(EXPEDITION) - - with open(out_path, "w") as file: - file.write(get_example_expedition()) - - expedition = Expedition.from_yaml(out_path) - - return expedition - - -@pytest.mark.usefixtures("copernicus_no_download") -def test_fetch(expedition, tmpdir): - """Test the fetch command, but mock the download and dataset metadata interrogation.""" - _fetch(Path(tmpdir), "test", "test") - - -def test_create_hash(): - assert len(create_hash("correct-length")) == 8 - assert create_hash("same") == create_hash("same") - assert create_hash("unique1") != create_hash("unique2") - - -def test_hash_filename_roundtrip(): - hash_ = create_hash("test") - assert filename_to_hash(hash_to_filename(hash_)) == hash_ - - -def test_hash_model(): - class TestModel(BaseModel): - a: int - b: str - - hash_model(TestModel(a=0, b="b")) - - -def test_complete_download(tmp_path): - # Setup - DownloadMetadata(download_complete=False).to_yaml(tmp_path / DOWNLOAD_METADATA) - - complete_download(tmp_path) - - assert_complete_download(tmp_path) - - -def test_assert_complete_download_complete(tmp_path): - # Setup - DownloadMetadata(download_complete=True).to_yaml(tmp_path / DOWNLOAD_METADATA) - - assert_complete_download(tmp_path) - - -def test_assert_complete_download_incomplete(tmp_path): - # Setup - DownloadMetadata(download_complete=False).to_yaml(tmp_path / DOWNLOAD_METADATA) - - with pytest.raises(IncompleteDownloadError): - assert_complete_download(tmp_path) - - -def test_assert_complete_download_missing(tmp_path): - with pytest.raises(IncompleteDownloadError): - assert_complete_download(tmp_path) - - -@pytest.fixture -def existing_data_folder(tmp_path, monkeypatch): - # Setup - folders = [ - "YYYYMMDD_HHMMSS_hash", - "YYYYMMDD_HHMMSS_hash2", - "some-invalid-data-folder", - "YYYYMMDD_HHMMSS_hash3", - ] - data_folder = tmp_path - monkeypatch.setattr( - "virtualship.cli._fetch.assert_complete_download", lambda x: None - ) - for f in folders: - (data_folder / f).mkdir() - yield data_folder - - -def test_get_existing_download(existing_data_folder): - assert isinstance(get_existing_download(existing_data_folder, "hash"), Path) - assert get_existing_download(existing_data_folder, "missing-hash") is None diff --git a/tests/expedition/test_do_expedition.py b/tests/cli/test_run.py similarity index 54% rename from tests/expedition/test_do_expedition.py rename to tests/cli/test_run.py index 0dbcd99a..be15961c 100644 --- a/tests/expedition/test_do_expedition.py +++ b/tests/cli/test_run.py @@ -2,11 +2,11 @@ from pytest import CaptureFixture -from virtualship.expedition import do_expedition +from virtualship.cli import _run -def test_do_expedition(capfd: CaptureFixture) -> None: - do_expedition("expedition_dir", input_data=Path("expedition_dir/input_data")) +def test_run(capfd: CaptureFixture) -> None: + _run("expedition_dir", input_data=Path("expedition_dir/input_data")) out, _ = capfd.readouterr() assert "Your expedition has concluded successfully!" in out, ( "Expedition did not complete successfully." From dc2f4d903b9a3bc93d23fc4da58cff3d9410e124 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Nov 2025 08:52:47 +0000 Subject: [PATCH 67/97] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/instruments/base.py | 2 +- src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 2 +- src/virtualship/instruments/ship_underwater_st.py | 2 +- src/virtualship/instruments/xbt.py | 2 +- src/virtualship/utils.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 8dca31a3..c53dbc1c 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 8ba794ba..09877471 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,7 +4,6 @@ from typing import ClassVar import numpy as np - from parcels import ( AdvectionRK4, JITParticle, @@ -12,6 +11,7 @@ StatusCode, Variable, ) + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 5c764cb3..fe698231 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -6,9 +6,9 @@ import copernicusmarine import xarray as xr +from parcels import FieldSet from yaspin import yaspin -from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 04fb24ed..4684c822 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import TYPE_CHECKING, ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 59dc4bc2..d3adcec3 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 8ad0fe19..3ee8ad1c 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index a28ae12d..6d8d4575 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_instrument diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 5972efff..df68122c 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index d44c528f..9474e5c4 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -11,8 +11,8 @@ import copernicusmarine import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: From 2a05ed45f597a799f1cc5ed34b04ecd85268d707 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Nov 2025 11:36:45 +0100 Subject: [PATCH 68/97] update tests (not yet instrument subclasses) --- src/virtualship/instruments/base.py | 3 + src/virtualship/static/expedition.yaml | 14 +- tests/cli/test_run.py | 65 +++++++- tests/conftest.py | 2 +- tests/expedition/test_expedition.py | 12 +- tests/instruments/test_base.py | 218 ++++++++++++------------- tests/test_utils.py | 210 ++++++++++++++++++++++-- 7 files changed, 374 insertions(+), 150 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 5c764cb3..2188332a 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -22,6 +22,9 @@ from virtualship.models import Expedition +# TODO: from-data should default to None and only be overwritten if specified in `virtualship run` ... + + class Instrument(abc.ABC): """Base class for instruments and their simulation.""" diff --git a/src/virtualship/static/expedition.yaml b/src/virtualship/static/expedition.yaml index 256bee87..2b770735 100644 --- a/src/virtualship/static/expedition.yaml +++ b/src/virtualship/static/expedition.yaml @@ -8,8 +8,8 @@ schedule: minimum_depth: 0 maximum_depth: 2000 time_range: - start_time: 2023-01-01 00:00:00 - end_time: 2023-02-01 00:00:00 + start_time: 1998-01-01 00:00:00 + end_time: 1998-02-01 00:00:00 waypoints: - instrument: - CTD @@ -17,30 +17,30 @@ schedule: location: latitude: 0 longitude: 0 - time: 2023-01-01 00:00:00 + time: 1998-01-01 00:00:00 - instrument: - DRIFTER - CTD location: latitude: 0.01 longitude: 0.01 - time: 2023-01-01 01:00:00 + time: 1998-01-01 01:00:00 - instrument: - ARGO_FLOAT location: latitude: 0.02 longitude: 0.02 - time: 2023-01-01 02:00:00 + time: 1998-01-01 02:00:00 - instrument: - XBT location: latitude: 0.03 longitude: 0.03 - time: 2023-01-01 03:00:00 + time: 1998-01-01 03:00:00 - location: latitude: 0.03 longitude: 0.03 - time: 2023-01-01 03:00:00 + time: 1998-01-01 03:00:00 instruments_config: adcp_config: num_bins: 40 diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py index be15961c..daa822f0 100644 --- a/tests/cli/test_run.py +++ b/tests/cli/test_run.py @@ -1,13 +1,64 @@ +from datetime import datetime from pathlib import Path -from pytest import CaptureFixture +from virtualship.cli._run import _run +from virtualship.expedition.simulate_schedule import ( + MeasurementsToSimulate, + ScheduleOk, +) +from virtualship.utils import EXPEDITION, get_example_expedition -from virtualship.cli import _run +def _simulate_schedule(projection, expedition): + """Return a trivial ScheduleOk with no measurements to simulate.""" + return ScheduleOk( + time=datetime.now(), measurements_to_simulate=MeasurementsToSimulate() + ) + + +class DummyInstrument: + """Dummy instrument class that just creates empty output directories.""" + + def __init__(self, expedition, directory, from_data=None): + """Initialize DummyInstrument.""" + self.expedition = expedition + self.directory = Path(directory) + self.from_data = from_data + + def execute(self, measurements, out_path): + """Mock execute method.""" + out_path = Path(out_path) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.mkdir(parents=True, exist_ok=True) + + +def test_run(tmp_path, monkeypatch): + """Testing as if using pre-downloaded, local data.""" + expedition_dir = tmp_path / "expedition_dir" + expedition_dir.mkdir() + (expedition_dir / EXPEDITION).write_text(get_example_expedition()) + + monkeypatch.setattr("virtualship.cli._run.simulate_schedule", _simulate_schedule) -def test_run(capfd: CaptureFixture) -> None: - _run("expedition_dir", input_data=Path("expedition_dir/input_data")) - out, _ = capfd.readouterr() - assert "Your expedition has concluded successfully!" in out, ( - "Expedition did not complete successfully." + monkeypatch.setattr( + "virtualship.models.InstrumentsConfig.verify", lambda self, expedition: None ) + monkeypatch.setattr( + "virtualship.models.Schedule.verify", lambda self, *args, **kwargs: None + ) + + monkeypatch.setattr( + "virtualship.cli._run.get_instrument_class", lambda itype: DummyInstrument + ) + + fake_data_dir = None + + _run(expedition_dir, from_data=fake_data_dir) + + results_dir = expedition_dir / "results" + + assert results_dir.exists() and results_dir.is_dir() + cost_file = results_dir / "cost.txt" + assert cost_file.exists() + content = cost_file.read_text() + assert "cost:" in content diff --git a/tests/conftest.py b/tests/conftest.py index 1b7a1de0..5ceac033 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -"""Test configuration that is ran for every test.""" +"""Test configuration that is run for every test.""" import pytest diff --git a/tests/expedition/test_expedition.py b/tests/expedition/test_expedition.py index 4bed35e7..d2021131 100644 --- a/tests/expedition/test_expedition.py +++ b/tests/expedition/test_expedition.py @@ -5,7 +5,6 @@ import pytest from virtualship.errors import InstrumentsConfigError, ScheduleError -from virtualship.expedition.do_expedition import _load_input_data from virtualship.models import ( Expedition, Location, @@ -57,7 +56,7 @@ def test_verify_schedule() -> None: ship_speed_knots = _get_expedition(expedition_dir).ship_config.ship_speed_knots - schedule.verify(ship_speed_knots, None) + schedule.verify(ship_speed_knots, ignore_land_test=True) def test_get_instruments() -> None: @@ -156,7 +155,7 @@ def test_get_instruments() -> None: ), True, ScheduleError, - "space_time_region not found in schedule, please define it to fetch the data.", + "space_time_region not found in schedule, please define it to proceed.", id="NoSpaceTimeRegion", ), ], @@ -165,16 +164,11 @@ def test_verify_schedule_errors( schedule: Schedule, check_space_time_region: bool, error, match ) -> None: expedition = _get_expedition(expedition_dir) - input_data = _load_input_data( - expedition_dir, - expedition, - input_data=Path("expedition_dir/input_data"), - ) with pytest.raises(error, match=match): schedule.verify( expedition.ship_config.ship_speed_knots, - input_data, + ignore_land_test=True, check_space_time_region=check_space_time_region, ) diff --git a/tests/instruments/test_base.py b/tests/instruments/test_base.py index 0a58efcd..272290e6 100644 --- a/tests/instruments/test_base.py +++ b/tests/instruments/test_base.py @@ -1,119 +1,14 @@ -import datetime -from pathlib import Path from unittest.mock import MagicMock, patch -import numpy as np -import pytest -import xarray as xr - -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType -from virtualship.models.space_time_region import ( - SpaceTimeRegion, - SpatialRange, - TimeRange, -) -from virtualship.utils import get_input_dataset_class - -# test dataclass, particle class, kernels, etc. are defined for each instrument +from virtualship.utils import get_instrument_class -def test_all_instruments_have_input_class(): +def test_all_instruments_have_instrument_class(): for instrument in InstrumentType: - input_class = get_input_dataset_class(instrument) - assert input_class is not None, f"No input_class for {instrument}" - - -# test InputDataset class - - -class DummyInputDataset(InputDataset): - """A minimal InputDataset subclass for testing purposes.""" - - def get_datasets_dict(self): - """Return a dummy datasets dict for testing.""" - return { - "dummy": { - "physical": True, - "variables": ["var1"], - "output_filename": "dummy.nc", - } - } - - -@pytest.fixture -def dummy_space_time_region(): - spatial_range = SpatialRange( - minimum_longitude=0, - maximum_longitude=1, - minimum_latitude=0, - maximum_latitude=1, - minimum_depth=0, - maximum_depth=10, - ) - base_time = datetime.datetime.strptime("1950-01-01", "%Y-%m-%d") - time_range = TimeRange( - start_time=base_time, - end_time=base_time + datetime.timedelta(hours=1), - ) - return SpaceTimeRegion( - spatial_range=spatial_range, - time_range=time_range, - ) - - -def test_dummyinputdataset_initialization(dummy_space_time_region): - ds = DummyInputDataset( - name="test", - latlon_buffer=0.5, - datetime_buffer=1, - min_depth=0, - max_depth=10, - data_dir=".", - credentials={"username": "u", "password": "p"}, - space_time_region=dummy_space_time_region, - ) - assert ds.name == "test" - assert ds.latlon_buffer == 0.5 - assert ds.datetime_buffer == 1 - assert ds.min_depth == 0 - assert ds.max_depth == 10 - assert ds.data_dir == "." - assert ds.credentials["username"] == "u" - - -@patch("virtualship.instruments.base.copernicusmarine.open_dataset") -@patch("virtualship.instruments.base.copernicusmarine.subset") -def test_download_data_calls_subset( - mock_subset, mock_open_dataset, dummy_space_time_region -): - """Test that download_data calls the subset function correctly, will also test Copernicus Marine product id search logic.""" - mock_open_dataset.return_value = xr.Dataset( - { - "time": ( - "time", - [ - np.datetime64("1993-01-01T00:00:00"), - np.datetime64("2023-01-01T01:00:00"), - ], - ) - } - ) - ds = DummyInputDataset( - name="test", - latlon_buffer=0.5, - datetime_buffer=1, - min_depth=0, - max_depth=10, - data_dir=".", - credentials={"username": "u", "password": "p"}, - space_time_region=dummy_space_time_region, - ) - ds.download_data() - assert mock_subset.called - - -# test Instrument class + instrument_class = get_instrument_class(instrument) + assert instrument_class is not None, f"No instrument_class for {instrument}" class DummyInstrument(Instrument): @@ -125,16 +20,18 @@ def simulate(self, data_dir, measurements, out_path): @patch("virtualship.instruments.base.FieldSet") -@patch("virtualship.instruments.base.get_existing_download") -@patch("virtualship.instruments.base.get_space_time_region_hash") -def test_load_input_data_calls(mock_hash, mock_get_download, mock_FieldSet): +@patch( + "virtualship.instruments.base._select_product_id", return_value="dummy_product_id" +) +@patch("virtualship.instruments.base.copernicusmarine") +def test_load_input_data(mock_copernicusmarine, mock_select_product_id, mock_FieldSet): """Test Instrument.load_input_data with mocks.""" - mock_hash.return_value = "hash" - mock_get_download.return_value = Path("/tmp/data") mock_fieldset = MagicMock() mock_FieldSet.from_netcdf.return_value = mock_fieldset + mock_FieldSet.from_xarray_dataset.return_value = mock_fieldset mock_fieldset.gridset.grids = [MagicMock(negate_depth=MagicMock())] mock_fieldset.__getitem__.side_effect = lambda k: MagicMock() + mock_copernicusmarine.open_dataset.return_value = MagicMock() dummy = DummyInstrument( name="test", expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), @@ -144,7 +41,96 @@ def test_load_input_data_calls(mock_hash, mock_get_download, mock_FieldSet): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=False, + from_data=None, ) fieldset = dummy.load_input_data() - assert mock_FieldSet.from_netcdf.called + assert mock_FieldSet.from_xarray_dataset.called + assert fieldset == mock_fieldset assert fieldset == mock_fieldset + + +def test_execute_calls_simulate(monkeypatch): + dummy = DummyInstrument( + name="test", + expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), + directory="/tmp", + filenames={"A": "a.nc"}, + variables={"A": "a"}, + add_bathymetry=False, + allow_time_extrapolation=False, + verbose_progress=True, + from_data=None, + ) + dummy.simulate = MagicMock() + dummy.execute([1, 2, 3], "/tmp/out") + dummy.simulate.assert_called_once() + + +def test_get_spec_value_buffer_and_limit(): + dummy = DummyInstrument( + name="test", + expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), + directory="/tmp", + filenames={"A": "a.nc"}, + variables={"A": "a"}, + add_bathymetry=False, + allow_time_extrapolation=False, + verbose_progress=False, + buffer_spec={"latlon": 5.0}, + limit_spec={"depth_min": 10.0}, + from_data=None, + ) + assert dummy._get_spec_value("buffer", "latlon", 0.0) == 5.0 + assert dummy._get_spec_value("limit", "depth_min", None) == 10.0 + assert dummy._get_spec_value("buffer", "missing", 42) == 42 + + +def test_generate_fieldset_combines_fields(monkeypatch): + dummy = DummyInstrument( + name="test", + expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), + directory="/tmp", + filenames={"A": "a.nc", "B": "b.nc"}, + variables={"A": "a", "B": "b"}, + add_bathymetry=False, + allow_time_extrapolation=False, + verbose_progress=False, + from_data=None, + ) + dummy.from_data = None + + monkeypatch.setattr(dummy, "_get_copernicus_ds", lambda physical, var: MagicMock()) + + fs_A = MagicMock() + fs_B = MagicMock() + fs_B.B = MagicMock() + monkeypatch.setattr( + "virtualship.instruments.base.FieldSet.from_xarray_dataset", + lambda ds, varmap, dims, mesh=None: fs_A if "A" in varmap else fs_B, + ) + monkeypatch.setattr(fs_A, "add_field", MagicMock()) + dummy._generate_fieldset() + fs_A.add_field.assert_called_once_with(fs_B.B) + + +def test_load_input_data_error(monkeypatch): + dummy = DummyInstrument( + name="test", + expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), + directory="/tmp", + filenames={"A": "a.nc"}, + variables={"A": "a"}, + add_bathymetry=False, + allow_time_extrapolation=False, + verbose_progress=False, + from_data=None, + ) + monkeypatch.setattr( + dummy, "_generate_fieldset", lambda: (_ for _ in ()).throw(Exception("fail")) + ) + import virtualship.errors + + try: + dummy.load_input_data() + except virtualship.errors.CopernicusCatalogueError as e: + assert "Failed to load input data" in str(e) diff --git a/tests/test_utils.py b/tests/test_utils.py index bb8208f6..b5350540 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,94 @@ +from pathlib import Path + +import numpy as np +import pytest +import xarray as xr + +import virtualship.utils +from parcels import FieldSet from virtualship.models.expedition import Expedition -from virtualship.utils import get_example_expedition +from virtualship.utils import ( + _find_nc_file_with_variable, + _get_bathy_data, + _select_product_id, + _start_end_in_product_timerange, + add_dummy_UV, + get_example_expedition, +) + + +@pytest.fixture +def expedition(tmp_file): + with open(tmp_file, "w") as file: + file.write(get_example_expedition()) + return Expedition.from_yaml(tmp_file) + + +@pytest.fixture +def dummy_spatial_range(): + class DummySpatialRange: + minimum_longitude = 0 + maximum_longitude = 1 + minimum_latitude = 0 + maximum_latitude = 1 + minimum_depth = 0 + maximum_depth = 4 + + return DummySpatialRange() + + +@pytest.fixture +def dummy_time_range(): + class DummyTimeRange: + start_time = "2020-01-01" + end_time = "2020-01-02" + + return DummyTimeRange() + + +@pytest.fixture +def dummy_space_time_region(dummy_spatial_range, dummy_time_range): + class DummySpaceTimeRegion: + spatial_range = dummy_spatial_range + time_range = dummy_time_range + + return DummySpaceTimeRegion() + + +@pytest.fixture +def dummy_instrument(): + class DummyInstrument: + pass + + return DummyInstrument() + + +@pytest.fixture +def copernicus_no_download(monkeypatch): + """Mock the copernicusmarine `subset` and `open_dataset` functions, approximating the reanalysis products.""" + + # mock for copernicusmarine.subset + def fake_download(output_filename, output_directory, **_): + Path(output_directory).joinpath(output_filename).touch() + + def fake_open_dataset(*args, **kwargs): + return xr.Dataset( + coords={ + "time": ( + "time", + [ + np.datetime64("1993-01-01"), + np.datetime64("2022-01-01"), + ], # mock up rough renanalysis period + ) + } + ) + + monkeypatch.setattr("virtualship.utils.copernicusmarine.subset", fake_download) + monkeypatch.setattr( + "virtualship.utils.copernicusmarine.open_dataset", fake_open_dataset + ) + yield def test_get_example_expedition(): @@ -14,17 +103,118 @@ def test_valid_example_expedition(tmp_path): Expedition.from_yaml(path) -def test_instrument_registry_updates(): +def test_instrument_registry_updates(dummy_instrument): from virtualship import utils - class DummyInputDataset: - pass + utils.register_instrument("DUMMY_TYPE")(dummy_instrument) - class DummyInstrument: - pass + assert utils.INSTRUMENT_CLASS_MAP["DUMMY_TYPE"] is dummy_instrument + + +def test_add_dummy_UV_adds_fields(): + fieldset = FieldSet.from_data({"T": 1}, {"lon": 0, "lat": 0}, mesh="spherical") + fieldset.__dict__.pop("U", None) + fieldset.__dict__.pop("V", None) + + # should not have U or V fields initially + assert "U" not in fieldset.__dict__ + assert "V" not in fieldset.__dict__ + + add_dummy_UV(fieldset) + + # now U and V should be present + assert "U" in fieldset.__dict__ + assert "V" in fieldset.__dict__ + + # should not raise error if U and V already present + add_dummy_UV(fieldset) + + +@pytest.mark.usefixtures("copernicus_no_download") +def test_select_product_id(expedition): + """Should return the physical reanalysis product id via the timings prescribed in the static schedule.yaml file.""" + result = _select_product_id( + physical=True, + schedule_start=expedition.schedule.space_time_region.time_range.start_time, + schedule_end=expedition.schedule.space_time_region.time_range.end_time, + username="test", + password="test", + ) + assert result == "cmems_mod_glo_phy_my_0.083deg_P1D-m" + + +@pytest.mark.usefixtures("copernicus_no_download") +def test_start_end_in_product_timerange(expedition): + """Should return True for valid range ass determined by the static schedule.yaml file.""" + assert _start_end_in_product_timerange( + selected_id="cmems_mod_glo_phy_my_0.083deg_P1D-m", + schedule_start=expedition.schedule.space_time_region.time_range.start_time, + schedule_end=expedition.schedule.space_time_region.time_range.end_time, + username="test", + password="test", + ) + + +def test_get_bathy_data_local(tmp_path, dummy_space_time_region): + """Test that _get_bathy_data returns a FieldSet when given a local directory for --from-data.""" + # dummy .nc file with 'deptho' variable + data = np.array([[1, 2], [3, 4]]) + ds = xr.Dataset( + { + "deptho": (("x", "y"), data), + }, + coords={ + "longitude": (("x", "y"), np.array([[0, 1], [0, 1]])), + "latitude": (("x", "y"), np.array([[0, 0], [1, 1]])), + }, + ) + nc_path = tmp_path / "dummy.nc" + ds.to_netcdf(nc_path) + + # should return a FieldSet + fieldset = _get_bathy_data(dummy_space_time_region, from_data=tmp_path) + assert isinstance(fieldset, FieldSet) + assert hasattr(fieldset, "bathymetry") + assert np.allclose(fieldset.bathymetry.data, data) + + +def test_get_bathy_data_copernicusmarine(monkeypatch, dummy_space_time_region): + """Test that _get_bathy_data calls copernicusmarine by default.""" + + def dummy_copernicusmarine(*args, **kwargs): + raise RuntimeError("copernicusmarine called") + + monkeypatch.setattr( + virtualship.utils.copernicusmarine, "open_dataset", dummy_copernicusmarine + ) + + try: + _get_bathy_data(dummy_space_time_region) + except RuntimeError as e: + assert "copernicusmarine called" in str(e) + + +def test_find_nc_file_with_variable_substring(tmp_path): + # dummy .nc file with variable 'uo_glor' (possible for CMS products to have similar suffixes...) + data = np.array([[1, 2], [3, 4]]) + ds = xr.Dataset( + { + "uo_glor": (("x", "y"), data), + }, + coords={ + "longitude": (("x", "y"), np.array([[0, 1], [0, 1]])), + "latitude": (("x", "y"), np.array([[0, 0], [1, 1]])), + }, + ) + nc_path = tmp_path / "test.nc" + ds.to_netcdf(nc_path) + + # should find 'uo_glor' when searching for 'uo' + result = _find_nc_file_with_variable(tmp_path, "uo") + assert result is not None + filename, found_var = result + assert filename == "test.nc" + assert found_var == "uo_glor" - utils.register_input_dataset("DUMMY_TYPE")(DummyInputDataset) - utils.register_instrument("DUMMY_TYPE")(DummyInstrument) - assert utils.INPUT_DATASET_MAP["DUMMY_TYPE"] is DummyInputDataset - assert utils.INSTRUMENT_CLASS_MAP["DUMMY_TYPE"] is DummyInstrument +# TODO: add test that pre-downloaded data is in correct directories - when have moved to be able to handle temporally separated .nc files! From 95a583d3e6ddef67f9b786f5895d492e33522f6d Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Nov 2025 13:25:03 +0100 Subject: [PATCH 69/97] update instrument tests --- src/virtualship/instruments/base.py | 4 +- src/virtualship/instruments/xbt.py | 2 + tests/instruments/test_adcp.py | 27 ++++++----- tests/instruments/test_argo_float.py | 23 +++++----- tests/instruments/test_ctd.py | 47 +++++++++++++++----- tests/instruments/test_ctd_bgc.py | 23 +++++----- tests/instruments/test_drifter.py | 24 +++++----- tests/instruments/test_ship_underwater_st.py | 26 ++++++----- tests/instruments/test_xbt.py | 23 +++++----- 9 files changed, 124 insertions(+), 75 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 2188332a..9996a670 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -24,6 +24,8 @@ # TODO: from-data should default to None and only be overwritten if specified in `virtualship run` ... +# TODO: update CMS credentials automation workflow so not all using the same credentials if running in a Jupyter Collaborative Session...! + class Instrument(abc.ABC): """Base class for instruments and their simulation.""" @@ -105,7 +107,7 @@ def simulate( def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" # - TMP = True # temporary spinner implementation + TMP = False # temporary spinner implementation # if TMP: if not self.verbose_progress: diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 5972efff..b9fe604f 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -150,6 +150,8 @@ def simulate(self, measurements, out_path) -> None: f"XBT max_depth or bathymetry shallower than minimum {-DT * fall_speed}. It is likely the XBT cannot be deployed in this area, which is too shallow." ) + breakpoint() + # define xbt particles xbt_particleset = ParticleSet( fieldset=fieldset, diff --git a/tests/instruments/test_adcp.py b/tests/instruments/test_adcp.py index 569f15a1..0f9003d6 100644 --- a/tests/instruments/test_adcp.py +++ b/tests/instruments/test_adcp.py @@ -4,9 +4,9 @@ import numpy as np import xarray as xr -from parcels import FieldSet -from virtualship.instruments.adcp import simulate_adcp +from parcels import FieldSet +from virtualship.instruments.adcp import ADCPInstrument from virtualship.models import Location, Spacetime @@ -77,17 +77,22 @@ def test_simulate_adcp(tmpdir) -> None: }, ) - # perform simulation + # dummy expedition and directory for ADCPInstrument + class DummyExpedition: + class instruments_config: + class adcp_config: + max_depth_meter = MAX_DEPTH + num_bins = NUM_BINS + + expedition = DummyExpedition() + directory = tmpdir + from_data = None + + adcp_instrument = ADCPInstrument(expedition, directory, from_data) out_path = tmpdir.join("out.zarr") - simulate_adcp( - fieldset=fieldset, - out_path=out_path, - max_depth=MAX_DEPTH, - min_depth=MIN_DEPTH, - num_bins=NUM_BINS, - sample_points=sample_points, - ) + adcp_instrument.load_input_data = lambda: fieldset + adcp_instrument.simulate(sample_points, out_path) results = xr.open_zarr(out_path) diff --git a/tests/instruments/test_argo_float.py b/tests/instruments/test_argo_float.py index 3eda53ae..045a7b7b 100644 --- a/tests/instruments/test_argo_float.py +++ b/tests/instruments/test_argo_float.py @@ -4,9 +4,9 @@ import numpy as np import xarray as xr -from parcels import FieldSet -from virtualship.instruments.argo_float import ArgoFloat, simulate_argo_floats +from parcels import FieldSet +from virtualship.instruments.argo_float import ArgoFloat, ArgoFloatInstrument from virtualship.models import Location, Spacetime @@ -53,16 +53,19 @@ def test_simulate_argo_floats(tmpdir) -> None: ) ] - # perform simulation + # dummy expedition and directory for ArgoFloatInstrument + class DummyExpedition: + pass + + expedition = DummyExpedition() + directory = tmpdir + from_data = None + + argo_instrument = ArgoFloatInstrument(expedition, directory, from_data) out_path = tmpdir.join("out.zarr") - simulate_argo_floats( - fieldset=fieldset, - out_path=out_path, - argo_floats=argo_floats, - outputdt=timedelta(minutes=5), - endtime=None, - ) + argo_instrument.load_input_data = lambda: fieldset + argo_instrument.simulate(argo_floats, out_path) # test if output is as expected results = xr.open_zarr(out_path) diff --git a/tests/instruments/test_ctd.py b/tests/instruments/test_ctd.py index 14e0a276..39b6cf47 100644 --- a/tests/instruments/test_ctd.py +++ b/tests/instruments/test_ctd.py @@ -5,13 +5,12 @@ """ import datetime -from datetime import timedelta import numpy as np import xarray as xr -from parcels import Field, FieldSet -from virtualship.instruments.ctd import CTD, simulate_ctd +from parcels import Field, FieldSet +from virtualship.instruments.ctd import CTD, CTDInstrument from virtualship.models import Location, Spacetime @@ -102,15 +101,42 @@ def test_simulate_ctds(tmpdir) -> None: ) fieldset.add_field(Field("bathymetry", [-1000], lon=0, lat=0)) - # perform simulation + # dummy expedition and directory for CTDInstrument + class DummyExpedition: + class schedule: + class space_time_region: + time_range = type( + "TimeRange", + (), + { + "start_time": fieldset.T.grid.time_origin.fulltime( + fieldset.T.grid.time_full[0] + ), + "end_time": fieldset.T.grid.time_origin.fulltime( + fieldset.T.grid.time_full[-1] + ), + }, + )() + spatial_range = type( + "SpatialRange", + (), + { + "minimum_longitude": 0, + "maximum_longitude": 1, + "minimum_latitude": 0, + "maximum_latitude": 1, + }, + )() + + expedition = DummyExpedition() + directory = tmpdir + from_data = None + + ctd_instrument = CTDInstrument(expedition, directory, from_data) out_path = tmpdir.join("out.zarr") - simulate_ctd( - ctds=ctds, - fieldset=fieldset, - out_path=out_path, - outputdt=timedelta(seconds=10), - ) + ctd_instrument.load_input_data = lambda: fieldset + ctd_instrument.simulate(ctds, out_path) # test if output is as expected results = xr.open_zarr(out_path) @@ -132,6 +158,7 @@ def test_simulate_ctds(tmpdir) -> None: for var in ["salinity", "temperature", "lat", "lon"]: obs_value = obs[var].values.item() exp_value = exp[var] + assert np.isclose(obs_value, exp_value), ( f"Observation incorrect {ctd_i=} {loc=} {var=} {obs_value=} {exp_value=}." ) diff --git a/tests/instruments/test_ctd_bgc.py b/tests/instruments/test_ctd_bgc.py index c1213884..4495f3e0 100644 --- a/tests/instruments/test_ctd_bgc.py +++ b/tests/instruments/test_ctd_bgc.py @@ -5,13 +5,12 @@ """ import datetime -from datetime import timedelta import numpy as np import xarray as xr -from parcels import Field, FieldSet -from virtualship.instruments.ctd_bgc import CTD_BGC, simulate_ctd_bgc +from parcels import Field, FieldSet +from virtualship.instruments.ctd_bgc import CTD_BGC, CTD_BGCInstrument from virtualship.models import Location, Spacetime @@ -163,15 +162,19 @@ def test_simulate_ctd_bgcs(tmpdir) -> None: ) fieldset.add_field(Field("bathymetry", [-1000], lon=0, lat=0)) - # perform simulation + # dummy expedition and directory for CTD_BGCInstrument + class DummyExpedition: + pass + + expedition = DummyExpedition() + directory = tmpdir + from_data = None + + ctd_bgc_instrument = CTD_BGCInstrument(expedition, directory, from_data) out_path = tmpdir.join("out.zarr") - simulate_ctd_bgc( - ctd_bgcs=ctd_bgcs, - fieldset=fieldset, - out_path=out_path, - outputdt=timedelta(seconds=10), - ) + ctd_bgc_instrument.load_input_data = lambda: fieldset + ctd_bgc_instrument.simulate(ctd_bgcs, out_path) # test if output is as expected results = xr.open_zarr(out_path) diff --git a/tests/instruments/test_drifter.py b/tests/instruments/test_drifter.py index ae230a87..095e6cdf 100644 --- a/tests/instruments/test_drifter.py +++ b/tests/instruments/test_drifter.py @@ -4,9 +4,9 @@ import numpy as np import xarray as xr -from parcels import FieldSet -from virtualship.instruments.drifter import Drifter, simulate_drifters +from parcels import FieldSet +from virtualship.instruments.drifter import Drifter, DrifterInstrument from virtualship.models import Location, Spacetime @@ -52,17 +52,19 @@ def test_simulate_drifters(tmpdir) -> None: ), ] - # perform simulation + # dummy expedition and directory for DrifterInstrument + class DummyExpedition: + pass + + expedition = DummyExpedition() + directory = tmpdir + from_data = None + + drifter_instrument = DrifterInstrument(expedition, directory, from_data) out_path = tmpdir.join("out.zarr") - simulate_drifters( - fieldset=fieldset, - out_path=out_path, - drifters=drifters, - outputdt=datetime.timedelta(hours=1), - dt=datetime.timedelta(minutes=5), - endtime=None, - ) + drifter_instrument.load_input_data = lambda: fieldset + drifter_instrument.simulate(drifters, out_path) # test if output is as expected results = xr.open_zarr(out_path) diff --git a/tests/instruments/test_ship_underwater_st.py b/tests/instruments/test_ship_underwater_st.py index 9d44ee6d..8e1cfbdc 100644 --- a/tests/instruments/test_ship_underwater_st.py +++ b/tests/instruments/test_ship_underwater_st.py @@ -4,16 +4,13 @@ import numpy as np import xarray as xr -from parcels import FieldSet -from virtualship.instruments.ship_underwater_st import simulate_ship_underwater_st +from parcels import FieldSet +from virtualship.instruments.ship_underwater_st import Underwater_STInstrument from virtualship.models import Location, Spacetime def test_simulate_ship_underwater_st(tmpdir) -> None: - # depth at which the sampling will be done - DEPTH = -2 - # arbitrary time offset for the dummy fieldset base_time = datetime.datetime.strptime("1950-01-01", "%Y-%m-%d") @@ -70,15 +67,20 @@ def test_simulate_ship_underwater_st(tmpdir) -> None: }, ) - # perform simulation + # dummy expedition and directory for Underwater_STInstrument + class DummyExpedition: + pass + + expedition = DummyExpedition() + directory = tmpdir + from_data = None + + st_instrument = Underwater_STInstrument(expedition, directory, from_data) out_path = tmpdir.join("out.zarr") - simulate_ship_underwater_st( - fieldset=fieldset, - out_path=out_path, - depth=DEPTH, - sample_points=sample_points, - ) + st_instrument.load_input_data = lambda: fieldset + # The instrument expects measurements as sample_points + st_instrument.simulate(sample_points, out_path) # test if output is as expected results = xr.open_zarr(out_path) diff --git a/tests/instruments/test_xbt.py b/tests/instruments/test_xbt.py index 97e33ade..7bafef9f 100644 --- a/tests/instruments/test_xbt.py +++ b/tests/instruments/test_xbt.py @@ -5,13 +5,12 @@ """ import datetime -from datetime import timedelta import numpy as np import xarray as xr -from parcels import Field, FieldSet -from virtualship.instruments.xbt import XBT, simulate_xbt +from parcels import Field, FieldSet +from virtualship.instruments.xbt import XBT, XBTInstrument from virtualship.models import Location, Spacetime @@ -96,15 +95,19 @@ def test_simulate_xbts(tmpdir) -> None: ) fieldset.add_field(Field("bathymetry", [-1000], lon=0, lat=0)) - # perform simulation + # dummy expedition and directory for XBTInstrument + class DummyExpedition: + pass + + expedition = DummyExpedition() + directory = tmpdir + from_data = None + + xbt_instrument = XBTInstrument(expedition, directory, from_data) out_path = tmpdir.join("out.zarr") - simulate_xbt( - xbts=xbts, - fieldset=fieldset, - out_path=out_path, - outputdt=timedelta(seconds=10), - ) + xbt_instrument.load_input_data = lambda: fieldset + xbt_instrument.simulate(xbts, out_path) # test if output is as expected results = xr.open_zarr(out_path) From 411c0427a8d9aff9245e5d0469870641ffe03fbe Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Nov 2025 14:42:24 +0100 Subject: [PATCH 70/97] test is on land tests in schedule.verify() --- src/virtualship/instruments/base.py | 26 +++------ src/virtualship/instruments/xbt.py | 2 - src/virtualship/models/expedition.py | 11 +++- tests/expedition/test_expedition.py | 82 ++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 22 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 9996a670..4eb8d042 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -106,26 +106,18 @@ def simulate( def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - # - TMP = False # temporary spinner implementation - # - if TMP: - if not self.verbose_progress: - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: - self.simulate(measurements, out_path) - spinner.ok("✅\n") - else: - print(f"Simulating {self.name} measurements... ") + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: self.simulate(measurements, out_path) - print("\n") - - # + spinner.ok("✅\n") else: + print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) + print("\n") def _get_copernicus_ds( self, diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index b9fe604f..5972efff 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -150,8 +150,6 @@ def simulate(self, measurements, out_path) -> None: f"XBT max_depth or bathymetry shallower than minimum {-DT * fall_speed}. It is likely the XBT cannot be deployed in this area, which is too shallow." ) - breakpoint() - # define xbt particles xbt_particleset = ParticleSet( fieldset=fieldset, diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 96e591b3..18846c1c 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import TYPE_CHECKING +import numpy as np import pydantic import pyproj import yaml @@ -145,14 +146,18 @@ def verify( for wp_i, wp in enumerate(self.waypoints): try: - bathymetry_field.eval( + value = bathymetry_field.eval( 0, # time 0, # depth (surface) wp.location.lat, wp.location.lon, ) - except Exception: - land_waypoints.append((wp_i, wp)) + if value == 0.0 or (isinstance(value, float) and np.isnan(value)): + land_waypoints.append((wp_i, wp)) + except Exception as e: + raise ScheduleError( + f"Waypoint #{wp_i + 1} at location {wp.location} could not be evaluated against bathymetry data. There may be a problem with the waypoint location being outside of the space_time_region or with the bathymetry data itself.\n\n Original error: {e}" + ) from e if len(land_waypoints) > 0: raise ScheduleError( diff --git a/tests/expedition/test_expedition.py b/tests/expedition/test_expedition.py index d2021131..9e08ecfc 100644 --- a/tests/expedition/test_expedition.py +++ b/tests/expedition/test_expedition.py @@ -1,9 +1,13 @@ from datetime import datetime, timedelta from pathlib import Path +from unittest.mock import patch +import numpy as np import pyproj import pytest +import xarray as xr +from parcels import FieldSet from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.models import ( Expedition, @@ -11,6 +15,11 @@ Schedule, Waypoint, ) +from virtualship.models.space_time_region import ( + SpaceTimeRegion, + SpatialRange, + TimeRange, +) from virtualship.utils import EXPEDITION, _get_expedition, get_example_expedition projection = pyproj.Geod(ellps="WGS84") @@ -85,6 +94,79 @@ def test_get_instruments() -> None: ) +def test_verify_on_land(): + """Test that schedule verification raises error for waypoints on land (0.0 m bathymetry).""" + # bathymetry fieldset with NaNs at specific locations + latitude = np.array([0, 1.0, 2.0]) + longitude = np.array([0, 1.0, 2.0]) + bathymetry = np.array( + [ + [100, 0.0, 100], + [100, 100, 0.0], + [0.0, 100, 100], + ] + ) + + ds_bathymetry = xr.Dataset( + { + "deptho": (("latitude", "longitude"), bathymetry), + }, + coords={ + "latitude": latitude, + "longitude": longitude, + }, + ) + + bathymetry_variables = {"bathymetry": "deptho"} + bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} + bathymetry_fieldset = FieldSet.from_xarray_dataset( + ds_bathymetry, bathymetry_variables, bathymetry_dimensions + ) + + # waypoints placed in NaN bathy cells + waypoints = [ + Waypoint( + location=Location(0.0, 1.0), time=datetime(2022, 1, 1, 1, 0, 0) + ), # NaN cell + Waypoint( + location=Location(1.0, 2.0), time=datetime(2022, 1, 2, 1, 0, 0) + ), # NaN cell + Waypoint( + location=Location(2.0, 0.0), time=datetime(2022, 1, 3, 1, 0, 0) + ), # NaN cell + ] + + spatial_range = SpatialRange( + minimum_latitude=min(wp.location.lat for wp in waypoints), + maximum_latitude=max(wp.location.lat for wp in waypoints), + minimum_longitude=min(wp.location.lon for wp in waypoints), + maximum_longitude=max(wp.location.lon for wp in waypoints), + ) + time_range = TimeRange( + start_time=min(wp.time for wp in waypoints if wp.time is not None), + end_time=max(wp.time for wp in waypoints if wp.time is not None), + ) + space_time_region = SpaceTimeRegion( + spatial_range=spatial_range, time_range=time_range + ) + schedule = Schedule(waypoints=waypoints, space_time_region=space_time_region) + ship_speed_knots = _get_expedition(expedition_dir).ship_config.ship_speed_knots + + with patch( + "virtualship.models.expedition._get_bathy_data", + return_value=bathymetry_fieldset, + ): + with pytest.raises( + ScheduleError, + match=r"The following waypoint\(s\) throw\(s\) error\(s\):", + ): + schedule.verify( + ship_speed_knots, + ignore_land_test=False, + from_data=None, + ) + + @pytest.mark.parametrize( "schedule,check_space_time_region,error,match", [ From e364d00cbdda77757f8b8a26afab59632da6cdd3 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Nov 2025 14:52:11 +0100 Subject: [PATCH 71/97] Run pre-commit --- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/instruments/base.py | 2 +- src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 2 +- src/virtualship/instruments/ship_underwater_st.py | 2 +- src/virtualship/instruments/xbt.py | 2 +- src/virtualship/utils.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index c53dbc1c..8dca31a3 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np -from parcels import ParticleSet, ScipyParticle, Variable +from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 09877471..8ba794ba 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,6 +4,7 @@ from typing import ClassVar import numpy as np + from parcels import ( AdvectionRK4, JITParticle, @@ -11,7 +12,6 @@ StatusCode, Variable, ) - from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index ccec2294..4eb8d042 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -6,9 +6,9 @@ import copernicusmarine import xarray as xr -from parcels import FieldSet from yaspin import yaspin +from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 4684c822..04fb24ed 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import TYPE_CHECKING, ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index d3adcec3..59dc4bc2 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 3ee8ad1c..8ad0fe19 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable +from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 6d8d4575..a28ae12d 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np -from parcels import ParticleSet, ScipyParticle, Variable +from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_instrument diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index df68122c..5972efff 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 9474e5c4..d44c528f 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -11,8 +11,8 @@ import copernicusmarine import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: From 56d41e6a4fdfd9939c2dada67ebd9a2379be0b39 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 12 Nov 2025 09:32:04 +0100 Subject: [PATCH 72/97] update pre-download ingestion methods to take files split by time --- src/virtualship/instruments/base.py | 102 +++++++++++++++------------ src/virtualship/models/expedition.py | 5 -- src/virtualship/utils.py | 8 ++- 3 files changed, 60 insertions(+), 55 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 4eb8d042..3ca1cdfa 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,6 +1,7 @@ import abc +import re from collections import OrderedDict -from datetime import timedelta +from datetime import datetime, timedelta from pathlib import Path from typing import TYPE_CHECKING @@ -156,51 +157,11 @@ def _get_copernicus_ds( coordinates_selection_method="outside", ) - def _load_local_ds(self, filename) -> xr.Dataset: - """ - Load local dataset from specified data directory. - - Sliced according to expedition.schedule.space_time_region and buffer specs. - """ - ds = xr.open_dataset(self.from_data.joinpath(filename)) - - coord_rename = {} - if "lat" in ds.coords: - coord_rename["lat"] = "latitude" - if "lon" in ds.coords: - coord_rename["lon"] = "longitude" - if coord_rename: - ds = ds.rename(coord_rename) - - min_lon = ( - self.expedition.schedule.space_time_region.spatial_range.minimum_longitude - - self._get_spec_value( - "buffer", "latlon", 3.0 - ) # always add min 3 deg buffer for local data to avoid edge issues with ds.sel() - ) - max_lon = ( - self.expedition.schedule.space_time_region.spatial_range.maximum_longitude - + self._get_spec_value("buffer", "latlon", 3.0) - ) - min_lat = ( - self.expedition.schedule.space_time_region.spatial_range.minimum_latitude - - self._get_spec_value("buffer", "latlon", 3.0) - ) - max_lat = ( - self.expedition.schedule.space_time_region.spatial_range.maximum_latitude - + self._get_spec_value("buffer", "latlon", 3.0) - ) - - return ds.sel( - latitude=slice(min_lat, max_lat), - longitude=slice(min_lon, max_lon), - ) - def _generate_fieldset(self) -> FieldSet: """ Create and combine FieldSets for each variable, supporting both local and Copernicus Marine data sources. - Avoids issues when using copernicusmarine and creating directly one FieldSet of ds's sourced from different Copernicus Marine product IDs, which is often the case for BGC variables. + Per variable avoids issues when using copernicusmarine and creating directly one FieldSet of ds's sourced from different Copernicus Marine product IDs, which is often the case for BGC variables. """ fieldsets_list = [] keys = list(self.variables.keys()) @@ -208,12 +169,34 @@ def _generate_fieldset(self) -> FieldSet: for key in keys: var = self.variables[key] if self.from_data is not None: # load from local data - filename, full_var_name = _find_nc_file_with_variable( - self.from_data, var + physical = var in COPERNICUSMARINE_PHYS_VARIABLES + if physical: + data_dir = self.from_data.joinpath("phys") + else: + data_dir = self.from_data.joinpath("bgc") + + schedule_start = ( + self.expedition.schedule.space_time_region.time_range.start_time ) - ds = self._load_local_ds(filename) - fs = FieldSet.from_xarray_dataset( - ds, {key: full_var_name}, self.dimensions, mesh="spherical" + schedule_end = ( + self.expedition.schedule.space_time_region.time_range.end_time + ) + + files = self._find_files_in_timerange( + data_dir, + schedule_start, + schedule_end, + ) + + _, full_var_name = _find_nc_file_with_variable( + data_dir, var + ) # get full variable name from one of the files; var may only appear as substring in variable name in file + + fs = FieldSet.from_netcdf( + filenames=[data_dir.joinpath(f) for f in files], + variables={key: full_var_name}, + dimensions=self.dimensions, + mesh="spherical", ) else: # steam via Copernicus Marine physical = var in COPERNICUSMARINE_PHYS_VARIABLES @@ -233,3 +216,28 @@ def _get_spec_value(self, spec_type: str, key: str, default=None): """Helper to extract a value from buffer_spec or limit_spec.""" spec = self.buffer_spec if spec_type == "buffer" else self.limit_spec return spec.get(key) if spec and spec.get(key) is not None else default + + def _find_files_in_timerange( + self, + data_dir: Path, + schedule_start, + schedule_end, + date_pattern=r"\d{4}_\d{2}_\d{2}", + date_fmt="%Y_%m_%d", + ) -> list: + """Find all files in data_dir whose filenames contain a date within [schedule_start, schedule_end] (inclusive).""" + # TODO: scope to make this more flexible for different date patterns / formats ... + files_with_dates = [] + start_date = schedule_start.date() # normalise to date only for comparison (given start/end dates have hour/minute components which may exceed those in file_date) + end_date = schedule_end.date() + for file in data_dir.iterdir(): + if file.is_file(): + match = re.search(date_pattern, file.name) + if match: + file_date = datetime.strptime(match.group(), date_fmt).date() + if start_date <= file_date <= end_date: + files_with_dates.append((file_date, file.name)) + files_with_dates.sort( + key=lambda x: x[0] + ) # sort by extracted date; more robust than relying on filesystem order + return [fname for _, fname in files_with_dates] diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 18846c1c..1ca5076d 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -3,7 +3,6 @@ import itertools from datetime import datetime, timedelta from pathlib import Path -from typing import TYPE_CHECKING import numpy as np import pydantic @@ -17,10 +16,6 @@ from .location import Location from .space_time_region import SpaceTimeRegion -if TYPE_CHECKING: - pass - - projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index d44c528f..e0ca1dc0 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -419,13 +419,15 @@ def _get_bathy_data( """Bathymetry data from local or 'streamed' directly from Copernicus Marine.""" if from_data is not None: # load from local data var = "deptho" + bathy_dir = from_data.joinpath("bathymetry/") try: - filename, _ = _find_nc_file_with_variable(from_data, var) + filename, _ = _find_nc_file_with_variable(bathy_dir, var) except Exception as e: + # TODO: link to documentation on expected data structure!! raise RuntimeError( - f"Could not find bathymetry variable '{var}' in provided data directory '{from_data}'." + f"\n\n❗️ Could not find bathymetry variable '{var}' in data directory '{from_data}/bathymetry/'.\n\n❗️ Is the pre-downloaded data directory structure compliant with VirtualShip expectations?\n\n❗️ See for more information on expectations: <<>>\n" ) from e - ds_bathymetry = xr.open_dataset(from_data.joinpath(filename)) + ds_bathymetry = xr.open_dataset(bathy_dir.joinpath(filename)) bathymetry_variables = {"bathymetry": "deptho"} bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} return FieldSet.from_xarray_dataset( From 845a8f8916c1f71b44d9dc1f876c025f29ac4407 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 13 Nov 2025 15:36:29 +0100 Subject: [PATCH 73/97] fix bug --- tests/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index b5350540..46bb1b78 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -168,7 +168,8 @@ def test_get_bathy_data_local(tmp_path, dummy_space_time_region): "latitude": (("x", "y"), np.array([[0, 0], [1, 1]])), }, ) - nc_path = tmp_path / "dummy.nc" + nc_path = tmp_path / "bathymetry/dummy.nc" + nc_path.parent.mkdir(parents=True, exist_ok=True) ds.to_netcdf(nc_path) # should return a FieldSet From 1418a446a7aa39829cb1a7512646ba828be84679 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 13 Nov 2025 15:36:48 +0100 Subject: [PATCH 74/97] fix bug in ingesting bgc data from disk --- src/virtualship/instruments/base.py | 81 +++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 15 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 3ca1cdfa..42391052 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,4 +1,5 @@ import abc +import glob import re from collections import OrderedDict from datetime import datetime, timedelta @@ -107,18 +108,24 @@ def simulate( def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - if not self.verbose_progress: - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: + TMP = True + + if TMP: + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate(measurements, out_path) + spinner.ok("✅\n") + else: + print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) - spinner.ok("✅\n") + print("\n") + else: - print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) - print("\n") def _get_copernicus_ds( self, @@ -192,8 +199,11 @@ def _generate_fieldset(self) -> FieldSet: data_dir, var ) # get full variable name from one of the files; var may only appear as substring in variable name in file - fs = FieldSet.from_netcdf( - filenames=[data_dir.joinpath(f) for f in files], + ds = xr.open_mfdataset( + [data_dir.joinpath(f) for f in files] + ) # as ds --> .from_xarray_dataset seems more robust than .from_netcdf for handling different temporal resolutions for different variables ... + fs = FieldSet.from_xarray_dataset( + ds, variables={key: full_var_name}, dimensions=self.dimensions, mesh="spherical", @@ -227,17 +237,58 @@ def _find_files_in_timerange( ) -> list: """Find all files in data_dir whose filenames contain a date within [schedule_start, schedule_end] (inclusive).""" # TODO: scope to make this more flexible for different date patterns / formats ... + + all_files = glob.glob(str(data_dir.joinpath("*"))) + if not all_files: + raise ValueError( + f"No files found in data directory {data_dir}. Please ensure the directory contains files with 'P1D' or 'P1M' in their names as per Copernicus Marine Product ID naming conventions." + ) + + if all("P1D" in s for s in all_files): + t_resolution = "daily" + elif all("P1M" in s for s in all_files): + t_resolution = "monthly" + else: + raise ValueError( + f"Could not determine time resolution from filenames in data directory. Please ensure all filenames in {data_dir} contain either 'P1D' (daily) or 'P1M' (monthly), " + f"as per the Copernicus Marine Product ID naming conventions." + ) + + if t_resolution == "monthly": + t_min = schedule_start.date() + t_max = ( + schedule_end.date() + + timedelta( + days=32 + ) # buffer to ensure fieldset end date is always longer than schedule end date for monthly data + ) + else: # daily + t_min = schedule_start.date() + t_max = schedule_end.date() + files_with_dates = [] - start_date = schedule_start.date() # normalise to date only for comparison (given start/end dates have hour/minute components which may exceed those in file_date) - end_date = schedule_end.date() for file in data_dir.iterdir(): if file.is_file(): match = re.search(date_pattern, file.name) if match: - file_date = datetime.strptime(match.group(), date_fmt).date() - if start_date <= file_date <= end_date: + file_date = datetime.strptime( + match.group(), date_fmt + ).date() # normalise to date only for comparison (given start/end dates have hour/minute components which may exceed those in file_date) + if t_min <= file_date <= t_max: files_with_dates.append((file_date, file.name)) + files_with_dates.sort( key=lambda x: x[0] ) # sort by extracted date; more robust than relying on filesystem order + + # catch if not enough data coverage found for the requested time range + if files_with_dates[-1][0] < schedule_end.date(): + raise ValueError( + f"Not enough data coverage found in {data_dir} for the requested time range {schedule_start} to {schedule_end}. " + f"Latest available data is for date {files_with_dates[-1][0]}." + f"If using monthly data, please ensure that the last month downloaded covers the schedule end date + 1 month." + f"See documentation for more details: <>" + # TODO: add link to relevant documentation! + ) + return [fname for _, fname in files_with_dates] From 5226a0257b66d4f90e01886b9956ead4e5b11eef Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 13 Nov 2025 16:38:16 +0100 Subject: [PATCH 75/97] tidy up --- docs/user-guide/quickstart.md | 2 -- src/virtualship/cli/_run.py | 2 +- src/virtualship/cli/commands.py | 1 - src/virtualship/cli/validator_utils.py | 1 - .../expedition/simulate_schedule.py | 3 +- src/virtualship/instruments/base.py | 34 +++++++------------ src/virtualship/models/expedition.py | 1 - 7 files changed, 14 insertions(+), 30 deletions(-) diff --git a/docs/user-guide/quickstart.md b/docs/user-guide/quickstart.md index 32dc0fdc..f4f232ad 100644 --- a/docs/user-guide/quickstart.md +++ b/docs/user-guide/quickstart.md @@ -151,5 +151,3 @@ It might take up to an hour to simulate the measurements depending on your choic Upon successfully completing the simulation, results from the expedition will be stored in the `EXPEDITION_NAME/results` directory, written as [Zarr](https://zarr.dev/) files. From here you can carry on your analysis (offline). We encourage you to explore and analyse these data using [Xarray](https://docs.xarray.dev/en/stable/). We also provide various further [VirtualShip tutorials](https://virtualship.readthedocs.io/en/latest/user-guide/tutorials/index.html) which provide examples of how to visualise data recorded by the VirtualShip instruments. - - diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index cf83da7d..06337cf3 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -51,7 +51,7 @@ def _run(expedition_dir: str | Path, from_data: Path | None = None) -> None: if from_data is None: # TODO: caution, if collaborative environments, will this mean everyone uses the same credentials file? - # TODO: may need to think about how to deal with this if using collaborative environments AND streaming data via copernicusmarine + # TODO: need to think about how to deal with this for when using collaborative environments AND streaming data via copernicusmarine COPERNICUS_CREDS_FILE = os.path.expandvars( "$HOME/.copernicusmarine/.copernicusmarine-credentials" ) diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index 3e485497..b5840f17 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -77,7 +77,6 @@ def plan(path): _plan(Path(path)) -# TODO: also add option to 'stream' via link to dir elsewhere, e.g. simlink or path to data stored elsewhere that isn't expedition dir! @click.command() @click.argument( "path", diff --git a/src/virtualship/cli/validator_utils.py b/src/virtualship/cli/validator_utils.py index 83239ac8..402e48b1 100644 --- a/src/virtualship/cli/validator_utils.py +++ b/src/virtualship/cli/validator_utils.py @@ -123,7 +123,6 @@ def make_validator(condition, reference, value_type): Therefore, reference values for the conditions cannot be fed in dynamically and necessitates 'hard-coding' the condition and reference value combination. At present, Pydantic models in VirtualShip only require gt/ge/lt/le relative to **0.0** so the 'reference' value is always checked as being == 0.0 Additional custom conditions can be 'hard-coded' as new condition and reference combinations if Pydantic model specifications change in the future and/or new instruments are added to VirtualShip etc. - TODO: Perhaps there's scope here though for a more flexible implementation in a future PR... """ diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 3b8fa78a..1d43dc13 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -122,8 +122,7 @@ def simulate(self) -> ScheduleOk | ScheduleProblem: # check if waypoint was reached in time if waypoint.time is not None and self._time > waypoint.time: print( - # TODO: I think this should be wp_i + 1, not wp_i; otherwise it will be off by one - f"Waypoint {wp_i} could not be reached in time. Current time: {self._time}. Waypoint time: {waypoint.time}." + f"Waypoint {wp_i + 1} could not be reached in time. Current time: {self._time}. Waypoint time: {waypoint.time}." ) return ScheduleProblem(self._time, wp_i) else: diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 42391052..54b45822 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -24,11 +24,6 @@ from virtualship.models import Expedition -# TODO: from-data should default to None and only be overwritten if specified in `virtualship run` ... - -# TODO: update CMS credentials automation workflow so not all using the same credentials if running in a Jupyter Collaborative Session...! - - class Instrument(abc.ABC): """Base class for instruments and their simulation.""" @@ -108,24 +103,18 @@ def simulate( def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - TMP = True - - if TMP: - if not self.verbose_progress: - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: - self.simulate(measurements, out_path) - spinner.ok("✅\n") - else: - print(f"Simulating {self.name} measurements... ") + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: self.simulate(measurements, out_path) - print("\n") - + spinner.ok("✅\n") else: + print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) + print("\n") def _get_copernicus_ds( self, @@ -201,7 +190,8 @@ def _generate_fieldset(self) -> FieldSet: ds = xr.open_mfdataset( [data_dir.joinpath(f) for f in files] - ) # as ds --> .from_xarray_dataset seems more robust than .from_netcdf for handling different temporal resolutions for different variables ... + ) # using: ds --> .from_xarray_dataset seems more robust than .from_netcdf for handling different temporal resolutions for different variables ... + fs = FieldSet.from_xarray_dataset( ds, variables={key: full_var_name}, @@ -236,7 +226,7 @@ def _find_files_in_timerange( date_fmt="%Y_%m_%d", ) -> list: """Find all files in data_dir whose filenames contain a date within [schedule_start, schedule_end] (inclusive).""" - # TODO: scope to make this more flexible for different date patterns / formats ... + # TODO: scope to make this more flexible for different date patterns / formats ... ? all_files = glob.glob(str(data_dir.joinpath("*"))) if not all_files: diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 1ca5076d..4847b10c 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -125,7 +125,6 @@ def verify( ) # check if all waypoints are in water using bathymetry data - # TODO: write test that checks that will flag when waypoint is on land!! [add to existing suite of fail .verify() tests in test_expedition.py] land_waypoints = [] if not ignore_land_test: try: From 36574b0ce0dd9f24887b4d41681c73ecf37afc68 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 13 Nov 2025 16:48:38 +0100 Subject: [PATCH 76/97] add test for data directory structure compliance --- src/virtualship/utils.py | 2 +- tests/test_utils.py | 27 ++++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index e0ca1dc0..e4a13403 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -419,7 +419,7 @@ def _get_bathy_data( """Bathymetry data from local or 'streamed' directly from Copernicus Marine.""" if from_data is not None: # load from local data var = "deptho" - bathy_dir = from_data.joinpath("bathymetry/") + bathy_dir = from_data.joinpath("bathymetry") try: filename, _ = _find_nc_file_with_variable(bathy_dir, var) except Exception as e: diff --git a/tests/test_utils.py b/tests/test_utils.py index 46bb1b78..87e402bb 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -218,4 +218,29 @@ def test_find_nc_file_with_variable_substring(tmp_path): assert found_var == "uo_glor" -# TODO: add test that pre-downloaded data is in correct directories - when have moved to be able to handle temporally separated .nc files! +def test_data_dir_structure_compliance(): + """ + Test that Instrument._generate_fieldset and _get_bathy_data use the expected subdirectory names. + + ('phys', 'bgc', 'bathymetry') for local data loading, as required by documentation. + + To avoid drift from what expecations are laid out in the docs. + """ + base_path = Path(__file__).parent.parent / "src/virtualship/instruments/base.py" + utils_path = Path(__file__).parent.parent / "src/virtualship/utils.py" + + base_code = base_path.read_text(encoding="utf-8") + utils_code = utils_path.read_text(encoding="utf-8") + + # Check for phys and bgc in Instrument._generate_fieldset + assert 'self.from_data.joinpath("phys")' in base_code, ( + "Expected 'phys' subdirectory not found in Instrument._generate_fieldset." + ) + assert 'self.from_data.joinpath("bgc")' in base_code, ( + "Expected 'bgc' subdirectory not found in Instrument._generate_fieldset." + ) + + # Check for bathymetry in _get_bathy_data + assert 'from_data.joinpath("bathymetry")' in utils_code, ( + "Expected 'bathymetry' subdirectory not found in _get_bathy_data." + ) From 4d9f5edf7f9ee3296e6e452ebe2b224d358399e5 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 13 Nov 2025 18:15:12 +0100 Subject: [PATCH 77/97] update docs --- .../example_copernicus_download.ipynb | 193 ++++++++++++++++++ .../documentation/pre_download_data.md | 69 +++++++ docs/user-guide/index.md | 2 + docs/user-guide/quickstart.md | 8 + 4 files changed, 272 insertions(+) create mode 100644 docs/user-guide/documentation/example_copernicus_download.ipynb create mode 100644 docs/user-guide/documentation/pre_download_data.md diff --git a/docs/user-guide/documentation/example_copernicus_download.ipynb b/docs/user-guide/documentation/example_copernicus_download.ipynb new file mode 100644 index 00000000..e5740405 --- /dev/null +++ b/docs/user-guide/documentation/example_copernicus_download.ipynb @@ -0,0 +1,193 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a48322c9", + "metadata": {}, + "source": [ + "# Example Copernicus data download \n", + "\n", + "This Notebook provides a rough example of how to download Copernicus Marine data using the Copernicus Marine API.\n", + "\n", + "This will download:\n", + "- Global bathymetry data (static)\n", + "- Global biogeochemical monthly data (0.25 degree hindcast)\n", + "- Global physical daily data (0.25 degree reanalysis)\n", + "\n", + "For a singular year (2023) and two months (June and July).\n", + "\n", + "This notebook is intended as a basic example only. Modifications will be needed to adapt this to your own use case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7f5a7cc7", + "metadata": {}, + "outputs": [], + "source": [ + "import copernicusmarine\n", + "import os\n", + "from datetime import datetime" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7279d5a", + "metadata": {}, + "outputs": [], + "source": [ + "YEAR = \"2023\"\n", + "MONTHS = [\"06\", \"07\"]\n", + "DAYS = [\n", + " \"01\",\n", + " \"02\",\n", + " \"03\",\n", + " \"04\",\n", + " \"05\",\n", + " \"06\",\n", + " \"07\",\n", + " \"08\",\n", + " \"09\",\n", + " \"10\",\n", + " \"11\",\n", + " \"12\",\n", + " \"13\",\n", + " \"14\",\n", + " \"15\",\n", + " \"16\",\n", + " \"17\",\n", + " \"18\",\n", + " \"19\",\n", + " \"20\",\n", + " \"21\",\n", + " \"22\",\n", + " \"23\",\n", + " \"24\",\n", + " \"25\",\n", + " \"26\",\n", + " \"27\",\n", + " \"28\",\n", + " \"29\",\n", + " \"30\",\n", + " \"31\",\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a583dba", + "metadata": {}, + "outputs": [], + "source": [ + "### PHYSICAL DAILY FILES\n", + "\n", + "os.chdir(\"~/data/phys/\")\n", + "DATASET_ID = \"cmems_mod_glo_phy-all_my_0.25deg_P1D-m\"\n", + "\n", + "for month in MONTHS:\n", + " for day in DAYS:\n", + " # check is valid date\n", + " try:\n", + " datetime(year=int(YEAR), month=int(month), day=int(day), hour=0)\n", + " except ValueError:\n", + " continue\n", + "\n", + " filename = f\"{DATASET_ID}_global_fulldepth_{YEAR}_{month}_{day}.nc\"\n", + "\n", + " if os.path.exists(filename):\n", + " print(f\"File {filename} already exists, skipping...\")\n", + " continue\n", + "\n", + " copernicusmarine.subset(\n", + " dataset_id=DATASET_ID,\n", + " variables=[\"uo_glor\", \"vo_glor\", \"thetao_glor\", \"so_glor\"],\n", + " minimum_longitude=-180,\n", + " maximum_longitude=179.75,\n", + " minimum_latitude=-80,\n", + " maximum_latitude=90,\n", + " start_datetime=f\"{YEAR}-{month}-{day}T00:00:00\",\n", + " end_datetime=f\"{YEAR}-{month}-{day}T00:00:00\",\n", + " minimum_depth=0.5057600140571594,\n", + " maximum_depth=5902.0576171875,\n", + " output_filename=filename,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89921772", + "metadata": {}, + "outputs": [], + "source": [ + "### BIOGEOCHEMICAL MONTHLY FILES\n", + "\n", + "os.chdir(\"~/data/bgc/\")\n", + "DATASET_ID = \"cmems_mod_glo_bgc_my_0.25deg_P1M-m\"\n", + "DAY = \"01\"\n", + "\n", + "for month in MONTHS:\n", + " try:\n", + " datetime(year=int(YEAR), month=int(month), day=int(DAY), hour=0)\n", + " except ValueError:\n", + " continue\n", + "\n", + " filename = f\"{DATASET_ID}_global_fulldepth_{YEAR}_{month}_{DAY}.nc\"\n", + "\n", + " if os.path.exists(filename):\n", + " print(f\"File {filename} already exists, skipping...\")\n", + " continue\n", + "\n", + " copernicusmarine.subset(\n", + " dataset_id=\"cmems_mod_glo_bgc_my_0.25deg_P1M-m\",\n", + " variables=[\"chl\", \"no3\", \"nppv\", \"o2\", \"ph\", \"phyc\", \"po4\"],\n", + " minimum_longitude=-180,\n", + " maximum_longitude=179.75,\n", + " minimum_latitude=-80,\n", + " maximum_latitude=90,\n", + " start_datetime=f\"{YEAR}-{month}-{DAY}T00:00:00\",\n", + " end_datetime=f\"{YEAR}-{month}-{DAY}T00:00:00\",\n", + " minimum_depth=0.5057600140571594,\n", + " maximum_depth=5902.05810546875,\n", + " output_filename=filename,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b5495c6", + "metadata": {}, + "outputs": [], + "source": [ + "### BATHYMETRY FILE\n", + "os.chdir(\"~/data/bathymetry/\")\n", + "DATASET_ID = \"cmems_mod_glo_phy_anfc_0.083deg_static\"\n", + "filename = \"cmems_mod_glo_phy_anfc_0.083deg_static_bathymetry.nc\"\n", + "\n", + "copernicusmarine.subset(\n", + " dataset_id=DATASET_ID,\n", + " dataset_part=\"bathy\",\n", + " variables=[\"deptho\"],\n", + " minimum_longitude=-180,\n", + " maximum_longitude=179.91668701171875,\n", + " minimum_latitude=-80,\n", + " maximum_latitude=90,\n", + " minimum_depth=0.49402499198913574,\n", + " maximum_depth=0.49402499198913574,\n", + " output_filename=filename,\n", + ")" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/user-guide/documentation/pre_download_data.md b/docs/user-guide/documentation/pre_download_data.md new file mode 100644 index 00000000..a1877443 --- /dev/null +++ b/docs/user-guide/documentation/pre_download_data.md @@ -0,0 +1,69 @@ +# Pre-downloading data + +By default, VirtualShip will automatically 'stream' data from the Copernicus Marine Service via their [copernicusmarine toolbox](https://github.com/mercator-ocean/copernicus-marine-toolbox?tab=readme-ov-file). However, for users with limited or unreliable internet connectivity, or those wishing to manage data locally, it is possible to pre-download the required datasets. + +As outlined in the [Quickstart Guide](../tutorials/quickstart.md), the `virtualship run` command supports an optional `--from-data` argument, which allows users to specify a local directory containing the necessary data files. + +Example Python code for automating data download from Copernicus Marine can be found in the [Example Copernicus Download Notebook](example_copernicus_download.ipynb). + +### Data requirements + +When using pre-downloaded data with VirtualShip, the software supports only: daily and monthly resolution physical and biogeochemical data, along with a static bathymetry file. + +In addition, all pre-downloaded data must be split into separate files per timestep (i.e. one .nc file per day or month). + +Further, VirtualShip expects pre-downloaded data to be organised in a specific directory & filename structure within the specified local data directory. The expected structure is as outlined in the subsequent sections. + +#### Directory structure + +Assuming the local data directory (as supplied in the `--from-data` argument) is named `data/`, the expected subdirectory structure is: + +```bash +. +└── data + ├── bathymetry # containing the singular bathymetry .nc file + ├── bgc # containing biogeochemical .nc files + └── phys # containing physical .nc files +``` + +#### Filename conventions + +Within these subdirectories, the expected filename conventions are: + +- Physical data files (in `data/phys/`) should be named as follows: + - `_.nc` + - e.g. `cmems_mod_glo_phy-all_my_0.25deg_P1D-m_1998_05_01.nc` +- Biogeochemical data files (in `data/bgc/`) should be named as follows: + - `_.nc` + - e.g. `cmems_mod_glo_bgc_my_0.25deg_P1M-m_1998_05_01.nc` +- Bathymetry data file (in `data/bathymetry/`) should be named as follows: + - `cmems_mod_glo_phy_anfc_0.083deg_static_bathymetry.nc` + +```{tip} +Careful to use an underscore (`_`) as the separator between date components in the filenames (i.e. `YYYY_MM_DD`). +``` + +```{note} +Using the `` in the filenames is vital in order to correctly identify the temporal resolution of the data (daily or monthly). The `P1D` in the example above indicates daily data, whereas `P1M` would indicate monthly data. + +See [here](https://help.marine.copernicus.eu/en/articles/6820094-how-is-the-nomenclature-of-copernicus-marine-data-defined#h_34a5a6f21d) for more information on Copernicus dataset nomenclature. + +See also our own [documentation](copernicus_products.md) on the Copernicus products used natively by VirtualShip when streaming data. +``` + +#### Further assumptions + +The following assumptions are also made about the data: + +1. All pre-downloaded data files must be in NetCDF format (`.nc`). +2. Physical data files must contain the following variables: `uo`, `vo`, `so`, `thetao` + - Or these strings must appear as substrings within the variable names (e.g. `uo_glor` is acceptable for `uo`). +3. If using BGC instruments (e.g. `CTD_BGC`), the relevant biogeochemical data files must contain the following variables: `o2`, `chl`, `no3`, `po4`, `nppv`, `ph`, `phyc`. + - Or these strings must appear as substrings within the variable names (e.g. `o2_glor` is acceptable for `o2`). +4. Bathymetry data files must contain a variable named `deptho`. + +#### Also of note + +1. Whilst not mandatory to use data downloaded only from Copernicus Marine (any existing data you may hold can be re-organised accordingly), the assumptions made by VirtualShip regarding directory structure and filename conventions are motivated by alignment with the Copernicus Marine's practices. + - If you want to use pre-existing data with VirtualShip, which you may have accessed from a different source, it is possible to do so by restructuring and/or renaming your data files as necessary. +2. The whole VirtualShip pre-downloaded data workflow should support global data or subsets thereof, provided the data files contain the necessary variables and are structured as outlined above. diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 2578fbea..fac1c26c 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -15,4 +15,6 @@ assignments/index :maxdepth: 1 documentation/copernicus_products.md +documentation/pre_download_data.md +documentation/example_copernicus_download.ipynb ``` diff --git a/docs/user-guide/quickstart.md b/docs/user-guide/quickstart.md index f4f232ad..b2cec51e 100644 --- a/docs/user-guide/quickstart.md +++ b/docs/user-guide/quickstart.md @@ -1,5 +1,13 @@ # VirtualShip Quickstart Guide 🚢 +```{warning} +This quickstart guide is currently out of date with the latest version of VirtualShip (v1.0.0). + +It will be updated soon. + +In particular, the `virtualship fetch` command is no longer supported. Instead, data fetching is now integrated into the `virtualship run` command. See [#226](https://github.com/Parcels-code/virtualship/pull/226) for more details in the meantime. +``` + Welcome to this Quickstart to using VirtualShip. In this guide we will conduct a virtual expedition in the North Sea. Note, however, that you can plan your own expedition anywhere in the global ocean and conduct whatever set of measurements you wish! This Quickstart is available as an instructional video below, or you can continue with the step-by-step guide. From e9f3b1647d9e511c76b3d6aef2fcade2da424648 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 13 Nov 2025 21:46:20 +0100 Subject: [PATCH 78/97] edits to docs --- .../documentation/copernicus_products.md | 10 +++---- .../example_copernicus_download.ipynb | 2 +- .../documentation/pre_download_data.md | 30 +++++++++++++++---- docs/user-guide/quickstart.md | 6 ++-- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/docs/user-guide/documentation/copernicus_products.md b/docs/user-guide/documentation/copernicus_products.md index 78361984..fec42d44 100644 --- a/docs/user-guide/documentation/copernicus_products.md +++ b/docs/user-guide/documentation/copernicus_products.md @@ -2,7 +2,7 @@ VirtualShip supports running experiments anywhere in the global ocean from 1993 through to the present day (and approximately two weeks into the future), using the suite of products available from the [Copernicus Marine Data Store](https://data.marine.copernicus.eu/products). -The data sourcing task is handled by the `virtualship fetch` command. The three products relied on by `fetch` to source data for all [VirtualShip instruments](https://virtualship.readthedocs.io/en/latest/user-guide/assignments/Research_proposal_intro.html#Measurement-Options) (both physical and biogeochemical) are: +The data sourcing task is handled by the `virtualship run` command, which in turn relies on the [copernicusmarine toolbox](https://github.com/mercator-ocean/copernicus-marine-toolbox?tab=readme-ov-file) for 'streaming' data from the Copernicus Marine Data Store. The three products relied on in `run` to source data for all [VirtualShip instruments](https://virtualship.readthedocs.io/en/latest/user-guide/assignments/Research_proposal_intro.html#Measurement-Options) (both physical and biogeochemical) are: 1. **Reanalysis** (or "hindcast" for biogeochemistry). 2. **Renalysis interim** (or "hindcast interim" for biogeochemistry). @@ -15,7 +15,7 @@ The Copernicus Marine Service describe the differences between the three product As a general rule of thumb the three different products span different periods across the historical period to present and are intended to allow for continuity across the previous ~ 30 years. ```{note} -The ethos for automated dataset selection in `virtualship fetch` is to prioritise the Reanalysis/Hindcast products where possible (the 'work horse'), then _interim products where possible for continuity, and finally filling the very near-present (and near-future) temporal range with the Analysis & Forecast products. +The ethos for automated dataset selection in `virtualship run` is to prioritise the Reanalysis/Hindcast products where possible (the 'work horse'), then _interim products where possible for continuity, and finally filling the very near-present (and near-future) temporal range with the Analysis & Forecast products. ``` ```{warning} @@ -24,13 +24,13 @@ In the rare situation where the start and end times of an expedition schedule sp ### Data availability -The following tables summarise which Copernicus product is selected by `virtualship fetch` per combination of time period and variable (see legend below). +The following tables summarise which Copernicus product is selected by `virtualship run` per combination of time period and variable (see legend below). For biogeochemical variables `ph` and `phyc`, monthly products are required for hindcast and hindcast interim periods. For all other variables, daily products are available. #### Physical products -| Period | Product ID | Temporal Resolution | Typical Years Covered | Variables | +| Period | Dataset ID | Temporal Resolution | Typical Years Covered | Variables | | :------------------ | :--------------------------------------- | :------------------ | :---------------------------------- | :------------------------- | | Reanalysis | `cmems_mod_glo_phy_my_0.083deg_P1D-m` | Daily | ~30 years ago to ~5 years ago | `uo`, `vo`, `so`, `thetao` | | Reanalysis Interim | `cmems_mod_glo_phy_myint_0.083deg_P1D-m` | Daily | ~5 years ago to ~2 months ago | `uo`, `vo`, `so`, `thetao` | @@ -40,7 +40,7 @@ For biogeochemical variables `ph` and `phyc`, monthly products are required for #### Biogeochemical products -| Period | Product ID | Temporal Resolution | Typical Years Covered | Variables | Notes | +| Period | Dataset ID | Temporal Resolution | Typical Years Covered | Variables | Notes | | :---------------------------- | :----------------------------------------- | :------------------ | :---------------------------------- | :-------------------------------- | :------------------------------------- | | Hindcast | `cmems_mod_glo_bgc_my_0.25deg_P1D-m` | Daily | ~30 years ago to ~5 years ago | `o2`, `chl`, `no3`, `po4`, `nppv` | Most BGC variables except `ph`, `phyc` | | Hindcast (monthly) | `cmems_mod_glo_bgc_my_0.25deg_P1M-m` | Monthly | ~30 years ago to ~5 years ago | `ph`, `phyc` | Only `ph`, `phyc` (monthly only) | diff --git a/docs/user-guide/documentation/example_copernicus_download.ipynb b/docs/user-guide/documentation/example_copernicus_download.ipynb index e5740405..1e630520 100644 --- a/docs/user-guide/documentation/example_copernicus_download.ipynb +++ b/docs/user-guide/documentation/example_copernicus_download.ipynb @@ -7,7 +7,7 @@ "source": [ "# Example Copernicus data download \n", "\n", - "This Notebook provides a rough example of how to download Copernicus Marine data using the Copernicus Marine API.\n", + "This notebook provides a rough, non-optimised example of how to download Copernicus Marine data using the `copernicusmarine` Python package.\n", "\n", "This will download:\n", "- Global bathymetry data (static)\n", diff --git a/docs/user-guide/documentation/pre_download_data.md b/docs/user-guide/documentation/pre_download_data.md index a1877443..f8d7e46a 100644 --- a/docs/user-guide/documentation/pre_download_data.md +++ b/docs/user-guide/documentation/pre_download_data.md @@ -1,10 +1,14 @@ # Pre-downloading data -By default, VirtualShip will automatically 'stream' data from the Copernicus Marine Service via their [copernicusmarine toolbox](https://github.com/mercator-ocean/copernicus-marine-toolbox?tab=readme-ov-file). However, for users with limited or unreliable internet connectivity, or those wishing to manage data locally, it is possible to pre-download the required datasets. +By default, VirtualShip will automatically 'stream' data from the Copernicus Marine Service via the [copernicusmarine toolbox](https://github.com/mercator-ocean/copernicus-marine-toolbox?tab=readme-ov-file). However, for users who wish to manage data locally, it is possible to pre-download the required datasets and feed them into VirtualShip simulations. -As outlined in the [Quickstart Guide](../tutorials/quickstart.md), the `virtualship run` command supports an optional `--from-data` argument, which allows users to specify a local directory containing the necessary data files. + -Example Python code for automating data download from Copernicus Marine can be found in the [Example Copernicus Download Notebook](example_copernicus_download.ipynb). +As outlined in the [Quickstart Guide](https://virtualship.readthedocs.io/en/latest/user-guide/quickstart.html), the `virtualship run` command supports an optional `--from-data` argument, which allows users to specify a local directory containing the necessary data files. + +```{tip} +See the [for example...](#for-example) section for an example data download workflow. +``` ### Data requirements @@ -12,6 +16,10 @@ When using pre-downloaded data with VirtualShip, the software supports only: dai In addition, all pre-downloaded data must be split into separate files per timestep (i.e. one .nc file per day or month). +```{note} +**Monthly data**: when using monthly data, ensure that your final .nc file download is for the month *after* your expedition schedule end date. This is to ensure that a Parcels FieldSet can be generated under-the-hood which fully covers the expedition period. For example, if your expedition runs from 1st May to 15th May, your final monthly data file should be in June. Daily data files only need to cover the expedition period exactly. +``` + Further, VirtualShip expects pre-downloaded data to be organised in a specific directory & filename structure within the specified local data directory. The expected structure is as outlined in the subsequent sections. #### Directory structure @@ -32,10 +40,10 @@ Within these subdirectories, the expected filename conventions are: - Physical data files (in `data/phys/`) should be named as follows: - `_.nc` - - e.g. `cmems_mod_glo_phy-all_my_0.25deg_P1D-m_1998_05_01.nc` + - e.g. `cmems_mod_glo_phy-all_my_0.25deg_P1D-m_1998_05_01.nc` and so on for each timestep. - Biogeochemical data files (in `data/bgc/`) should be named as follows: - `_.nc` - - e.g. `cmems_mod_glo_bgc_my_0.25deg_P1M-m_1998_05_01.nc` + - e.g. `cmems_mod_glo_bgc_my_0.25deg_P1M-m_1998_05_01.nc` and so on for each timestep. - Bathymetry data file (in `data/bathymetry/`) should be named as follows: - `cmems_mod_glo_phy_anfc_0.083deg_static_bathymetry.nc` @@ -48,7 +56,11 @@ Using the `` in the filenames is vital in order See [here](https://help.marine.copernicus.eu/en/articles/6820094-how-is-the-nomenclature-of-copernicus-marine-data-defined#h_34a5a6f21d) for more information on Copernicus dataset nomenclature. -See also our own [documentation](copernicus_products.md) on the Copernicus products used natively by VirtualShip when streaming data. +See also our own [documentation](https://virtualship.readthedocs.io/en/latest/user-guide/documentation/copernicus_products.html#data-availability) on the Copernicus datasets used natively by VirtualShip when 'streaming' data if you wish to use the same datasets for pre-download. +``` + +```{note} +**Monthly data**: the `DD` component of the date in the filename for monthly .nc files should always be `01`, representing the first day of the month. This ensures that a Parcels FieldSet can be generated under-the-hood which fully covers the expedition period from the start. ``` #### Further assumptions @@ -67,3 +79,9 @@ The following assumptions are also made about the data: 1. Whilst not mandatory to use data downloaded only from Copernicus Marine (any existing data you may hold can be re-organised accordingly), the assumptions made by VirtualShip regarding directory structure and filename conventions are motivated by alignment with the Copernicus Marine's practices. - If you want to use pre-existing data with VirtualShip, which you may have accessed from a different source, it is possible to do so by restructuring and/or renaming your data files as necessary. 2. The whole VirtualShip pre-downloaded data workflow should support global data or subsets thereof, provided the data files contain the necessary variables and are structured as outlined above. + +#### For example... + +Example Python code for automating the data download from Copernicus Marine can be found in [Example Copernicus Download](example_copernicus_download.ipynb). + + diff --git a/docs/user-guide/quickstart.md b/docs/user-guide/quickstart.md index b2cec51e..2d648723 100644 --- a/docs/user-guide/quickstart.md +++ b/docs/user-guide/quickstart.md @@ -1,11 +1,9 @@ # VirtualShip Quickstart Guide 🚢 ```{warning} -This quickstart guide is currently out of date with the latest version of VirtualShip (v1.0.0). +This quickstart guide is designed for use with VirtualShip v0.2.2 and currently out of date with the latest version of VirtualShip (v1.0.0). It will be updated soon. -It will be updated soon. - -In particular, the `virtualship fetch` command is no longer supported. Instead, data fetching is now integrated into the `virtualship run` command. See [#226](https://github.com/Parcels-code/virtualship/pull/226) for more details in the meantime. +In particular, the `virtualship fetch` command is no longer supported. Instead, data fetching is now integrated into the `virtualship run` command. See [#226](https://github.com/Parcels-code/virtualship/pull/226) for details in the meantime. ``` Welcome to this Quickstart to using VirtualShip. In this guide we will conduct a virtual expedition in the North Sea. Note, however, that you can plan your own expedition anywhere in the global ocean and conduct whatever set of measurements you wish! From 554f2814eca220a67ee25ff4771ed25c386da917 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 13 Nov 2025 22:00:25 +0100 Subject: [PATCH 79/97] add more checks to docs compliance testing --- tests/test_utils.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 87e402bb..ba13447a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -218,13 +218,18 @@ def test_find_nc_file_with_variable_substring(tmp_path): assert found_var == "uo_glor" -def test_data_dir_structure_compliance(): +def test_data_dir_and_filename_compliance(): """ - Test that Instrument._generate_fieldset and _get_bathy_data use the expected subdirectory names. + Test compliance of data directory structure and filename patterns as sought by base.py methods relative to as is described in the docs. + + Test that: + - Instrument._generate_fieldset and _get_bathy_data use the expected subdirectory names. + - The expected filename date pattern (YYYY_MM_DD) is used in _find_files_in_timerange. + ('phys', 'bgc', 'bathymetry') for local data loading, as required by documentation. - To avoid drift from what expecations are laid out in the docs. + To avoid drift between code implementation and what expectations are laid out in the docs. """ base_path = Path(__file__).parent.parent / "src/virtualship/instruments/base.py" utils_path = Path(__file__).parent.parent / "src/virtualship/utils.py" @@ -234,13 +239,26 @@ def test_data_dir_structure_compliance(): # Check for phys and bgc in Instrument._generate_fieldset assert 'self.from_data.joinpath("phys")' in base_code, ( - "Expected 'phys' subdirectory not found in Instrument._generate_fieldset." + "Expected 'phys' subdirectory not found in Instrument._generate_fieldset. This indicates a drift between docs and implementation." ) assert 'self.from_data.joinpath("bgc")' in base_code, ( - "Expected 'bgc' subdirectory not found in Instrument._generate_fieldset." + "Expected 'bgc' subdirectory not found in Instrument._generate_fieldset. This indicates a drift between docs and implementation." ) # Check for bathymetry in _get_bathy_data assert 'from_data.joinpath("bathymetry")' in utils_code, ( - "Expected 'bathymetry' subdirectory not found in _get_bathy_data." + "Expected 'bathymetry' subdirectory not found in _get_bathy_data. This indicates a drift between docs and implementation." + ) + + # Check for date_pattern in _find_files_in_timerange + assert 'date_pattern=r"\\d{4}_\\d{2}_\\d{2}"' in base_code, ( + "Expected date_pattern r'\\d{4}_\\d{2}_\\d{2}' not found in _find_files_in_timerange. This indicates a drift between docs and implementation." + ) + + # Check for P1D and P1M in t_resolution logic + assert 'if all("P1D" in s for s in all_files):' in base_code, ( + "Expected check for 'P1D' in all_files not found in _find_files_in_timerange. This indicates a drift between docs and implementation." + ) + assert 'elif all("P1M" in s for s in all_files):' in base_code, ( + "Expected check for 'P1M' in all_files not found in _find_files_in_timerange. This indicates a drift between docs and implementation." ) From 48937311ab840b5e855c3d67ef46f3608e7ac117 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 14 Nov 2025 08:48:26 +0100 Subject: [PATCH 80/97] TODO in readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b9a59e70..c6ed488c 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ + + VirtualShipParcels is a command line simulator allowing students to plan and conduct a virtual research expedition, receiving measurements as if they were coming from actual oceanographic instruments including: From 5a6154167bc42179898e2148fcf0e264c0d76e88 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 14 Nov 2025 07:49:02 +0000 Subject: [PATCH 81/97] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/instruments/base.py | 2 +- src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 2 +- src/virtualship/instruments/ship_underwater_st.py | 2 +- src/virtualship/instruments/xbt.py | 2 +- src/virtualship/utils.py | 2 +- tests/expedition/test_expedition.py | 2 +- tests/instruments/test_adcp.py | 2 +- tests/instruments/test_argo_float.py | 2 +- tests/instruments/test_ctd.py | 2 +- tests/instruments/test_ctd_bgc.py | 2 +- tests/instruments/test_drifter.py | 2 +- tests/instruments/test_ship_underwater_st.py | 2 +- tests/instruments/test_xbt.py | 2 +- tests/test_utils.py | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 8dca31a3..c53dbc1c 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 8ba794ba..09877471 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,7 +4,6 @@ from typing import ClassVar import numpy as np - from parcels import ( AdvectionRK4, JITParticle, @@ -12,6 +11,7 @@ StatusCode, Variable, ) + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 54b45822..b5b46abd 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -8,9 +8,9 @@ import copernicusmarine import xarray as xr +from parcels import FieldSet from yaspin import yaspin -from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 04fb24ed..4684c822 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import TYPE_CHECKING, ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 59dc4bc2..d3adcec3 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 8ad0fe19..3ee8ad1c 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index a28ae12d..6d8d4575 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_instrument diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 5972efff..df68122c 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index e4a13403..84839b76 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -11,8 +11,8 @@ import copernicusmarine import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: diff --git a/tests/expedition/test_expedition.py b/tests/expedition/test_expedition.py index 9e08ecfc..78fff2c2 100644 --- a/tests/expedition/test_expedition.py +++ b/tests/expedition/test_expedition.py @@ -6,8 +6,8 @@ import pyproj import pytest import xarray as xr - from parcels import FieldSet + from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.models import ( Expedition, diff --git a/tests/instruments/test_adcp.py b/tests/instruments/test_adcp.py index 0f9003d6..f7802dc7 100644 --- a/tests/instruments/test_adcp.py +++ b/tests/instruments/test_adcp.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.adcp import ADCPInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_argo_float.py b/tests/instruments/test_argo_float.py index 045a7b7b..f974e19b 100644 --- a/tests/instruments/test_argo_float.py +++ b/tests/instruments/test_argo_float.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.argo_float import ArgoFloat, ArgoFloatInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_ctd.py b/tests/instruments/test_ctd.py index 39b6cf47..cde67a90 100644 --- a/tests/instruments/test_ctd.py +++ b/tests/instruments/test_ctd.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr - from parcels import Field, FieldSet + from virtualship.instruments.ctd import CTD, CTDInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_ctd_bgc.py b/tests/instruments/test_ctd_bgc.py index 4495f3e0..c5836542 100644 --- a/tests/instruments/test_ctd_bgc.py +++ b/tests/instruments/test_ctd_bgc.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr - from parcels import Field, FieldSet + from virtualship.instruments.ctd_bgc import CTD_BGC, CTD_BGCInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_drifter.py b/tests/instruments/test_drifter.py index 095e6cdf..d160a09f 100644 --- a/tests/instruments/test_drifter.py +++ b/tests/instruments/test_drifter.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.drifter import Drifter, DrifterInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_ship_underwater_st.py b/tests/instruments/test_ship_underwater_st.py index 8e1cfbdc..e5acdd0e 100644 --- a/tests/instruments/test_ship_underwater_st.py +++ b/tests/instruments/test_ship_underwater_st.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.ship_underwater_st import Underwater_STInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_xbt.py b/tests/instruments/test_xbt.py index 7bafef9f..6c33a6dc 100644 --- a/tests/instruments/test_xbt.py +++ b/tests/instruments/test_xbt.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr - from parcels import Field, FieldSet + from virtualship.instruments.xbt import XBT, XBTInstrument from virtualship.models import Location, Spacetime diff --git a/tests/test_utils.py b/tests/test_utils.py index ba13447a..3018411b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,9 +3,9 @@ import numpy as np import pytest import xarray as xr +from parcels import FieldSet import virtualship.utils -from parcels import FieldSet from virtualship.models.expedition import Expedition from virtualship.utils import ( _find_nc_file_with_variable, From 00c4455dde279cd07b1c5ce883763b5b14a5c828 Mon Sep 17 00:00:00 2001 From: Jamie Atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 14 Nov 2025 15:29:02 +0100 Subject: [PATCH 82/97] Update docs/user-guide/documentation/pre_download_data.md Co-authored-by: Erik van Sebille --- docs/user-guide/documentation/pre_download_data.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide/documentation/pre_download_data.md b/docs/user-guide/documentation/pre_download_data.md index f8d7e46a..4e49b21c 100644 --- a/docs/user-guide/documentation/pre_download_data.md +++ b/docs/user-guide/documentation/pre_download_data.md @@ -12,7 +12,7 @@ See the [for example...](#for-example) section for an example data download work ### Data requirements -When using pre-downloaded data with VirtualShip, the software supports only: daily and monthly resolution physical and biogeochemical data, along with a static bathymetry file. +For pre-downloaded data, VirtualShip only supports daily and monthly resolution physical and biogeochemical data, along with a static bathymetry file. In addition, all pre-downloaded data must be split into separate files per timestep (i.e. one .nc file per day or month). From 3cdfd1a4c5420c88fa19a827e6c38d0b4bf531d6 Mon Sep 17 00:00:00 2001 From: Jamie Atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 14 Nov 2025 15:32:07 +0100 Subject: [PATCH 83/97] Apply suggestions from code review Co-authored-by: Erik van Sebille --- docs/user-guide/documentation/pre_download_data.md | 4 ++-- src/virtualship/instruments/base.py | 2 +- src/virtualship/utils.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/user-guide/documentation/pre_download_data.md b/docs/user-guide/documentation/pre_download_data.md index 4e49b21c..fc55f339 100644 --- a/docs/user-guide/documentation/pre_download_data.md +++ b/docs/user-guide/documentation/pre_download_data.md @@ -48,7 +48,7 @@ Within these subdirectories, the expected filename conventions are: - `cmems_mod_glo_phy_anfc_0.083deg_static_bathymetry.nc` ```{tip} -Careful to use an underscore (`_`) as the separator between date components in the filenames (i.e. `YYYY_MM_DD`). +Take care to use an underscore (`_`) as the separator between date components in the filenames (i.e. `YYYY_MM_DD`). ``` ```{note} @@ -76,7 +76,7 @@ The following assumptions are also made about the data: #### Also of note -1. Whilst not mandatory to use data downloaded only from Copernicus Marine (any existing data you may hold can be re-organised accordingly), the assumptions made by VirtualShip regarding directory structure and filename conventions are motivated by alignment with the Copernicus Marine's practices. +1. Whilst not mandatory to use data downloaded only from the Copernicus Marine Service (any existing data you may hold can be re-organised accordingly), the assumptions made by VirtualShip regarding directory structure and filename conventions are motivated by alignment with the Copernicus Marine Service's practices. - If you want to use pre-existing data with VirtualShip, which you may have accessed from a different source, it is possible to do so by restructuring and/or renaming your data files as necessary. 2. The whole VirtualShip pre-downloaded data workflow should support global data or subsets thereof, provided the data files contain the necessary variables and are structured as outlined above. diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index b5b46abd..14a2179a 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -198,7 +198,7 @@ def _generate_fieldset(self) -> FieldSet: dimensions=self.dimensions, mesh="spherical", ) - else: # steam via Copernicus Marine + else: # stream via Copernicus Marine Service physical = var in COPERNICUSMARINE_PHYS_VARIABLES ds = self._get_copernicus_ds(physical=physical, var=var) fs = FieldSet.from_xarray_dataset( diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 84839b76..a8f61b49 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -434,7 +434,7 @@ def _get_bathy_data( ds_bathymetry, bathymetry_variables, bathymetry_dimensions ) - else: # stream via Copernicus Marine + else: # stream via Copernicus Marine Service ds_bathymetry = copernicusmarine.open_dataset( dataset_id="cmems_mod_glo_phy_my_0.083deg_static", minimum_longitude=space_time_region.spatial_range.minimum_longitude From a84c8fd479e424be4a0583c364e9e2584df3d9b9 Mon Sep 17 00:00:00 2001 From: Jamie Atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 18 Nov 2025 13:26:29 +0100 Subject: [PATCH 84/97] Set t_min to first day of month for monthly resolution Adjust t_min to the first day of the month based on schedule start date. --- src/virtualship/instruments/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 14a2179a..282babe8 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -245,7 +245,9 @@ def _find_files_in_timerange( ) if t_resolution == "monthly": - t_min = schedule_start.date() + t_min = schedule_start.date().replace( + day=1 + ) # first day of month of the schedule start date t_max = ( schedule_end.date() + timedelta( From 4198406acf0de6ee2ceed1ead80457376758e350 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 09:33:26 +0100 Subject: [PATCH 85/97] remove redundant parameters from instrument classes --- src/virtualship/cli/_run.py | 1 - src/virtualship/instruments/adcp.py | 11 ++--------- src/virtualship/instruments/argo_float.py | 13 ++----------- src/virtualship/instruments/base.py | 18 +++++++----------- src/virtualship/instruments/ctd.py | 11 ++--------- src/virtualship/instruments/ctd_bgc.py | 16 ++-------------- src/virtualship/instruments/drifter.py | 12 ++---------- .../instruments/ship_underwater_st.py | 11 ++--------- src/virtualship/instruments/xbt.py | 13 ++----------- tests/cli/test_run.py | 3 +-- tests/instruments/test_adcp.py | 6 +++--- tests/instruments/test_argo_float.py | 5 ++--- tests/instruments/test_base.py | 15 --------------- tests/instruments/test_ctd.py | 6 +++--- tests/instruments/test_ctd_bgc.py | 5 ++--- tests/instruments/test_drifter.py | 6 +++--- tests/instruments/test_ship_underwater_st.py | 6 +++--- tests/instruments/test_xbt.py | 6 +++--- 18 files changed, 41 insertions(+), 123 deletions(-) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index 06337cf3..6ae7a1cd 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -142,7 +142,6 @@ def _run(expedition_dir: str | Path, from_data: Path | None = None) -> None: # initialise instrument instrument = instrument_class( expedition=expedition, - directory=expedition_dir, from_data=Path(from_data) if from_data is not None else None, ) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index c53dbc1c..46f34faf 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np -from parcels import ParticleSet, ScipyParticle, Variable +from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( @@ -54,19 +54,12 @@ def _sample_velocity(particle, fieldset, time): class ADCPInstrument(Instrument): """ADCP instrument class.""" - def __init__(self, expedition, directory, from_data): + def __init__(self, expedition, from_data): """Initialize ADCPInstrument.""" - filenames = { - "U": f"{ADCP.name}_uv.nc", - "V": f"{ADCP.name}_uv.nc", - } variables = {"U": "uo", "V": "vo"} super().__init__( - ADCP.name, expedition, - directory, - filenames, variables, add_bathymetry=False, allow_time_extrapolation=True, diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 09877471..69095da9 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,6 +4,7 @@ from typing import ClassVar import numpy as np + from parcels import ( AdvectionRK4, JITParticle, @@ -11,7 +12,6 @@ StatusCode, Variable, ) - from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -137,14 +137,8 @@ def _check_error(particle, fieldset, time): class ArgoFloatInstrument(Instrument): """ArgoFloat instrument class.""" - def __init__(self, expedition, directory, from_data): + def __init__(self, expedition, from_data): """Initialize ArgoFloatInstrument.""" - filenames = { - "U": f"{ArgoFloat.name}_uv.nc", - "V": f"{ArgoFloat.name}_uv.nc", - "S": f"{ArgoFloat.name}_s.nc", - "T": f"{ArgoFloat.name}_t.nc", - } variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} buffer_spec = { "latlon": 3.0, # [degrees] @@ -152,10 +146,7 @@ def __init__(self, expedition, directory, from_data): } super().__init__( - ArgoFloat.name, expedition, - directory, - filenames, variables, add_bathymetry=False, allow_time_extrapolation=False, diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 282babe8..d9339942 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import abc import glob import re @@ -8,9 +10,9 @@ import copernicusmarine import xarray as xr -from parcels import FieldSet from yaspin import yaspin +from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, @@ -29,10 +31,7 @@ class Instrument(abc.ABC): def __init__( self, - name: str, - expedition: "Expedition", - directory: Path | str, - filenames: dict, + expedition: Expedition, variables: dict, add_bathymetry: bool, allow_time_extrapolation: bool, @@ -42,10 +41,7 @@ def __init__( limit_spec: dict | None = None, ): """Initialise instrument.""" - self.name = name self.expedition = expedition - self.directory = directory - self.filenames = filenames self.from_data = from_data self.variables = OrderedDict(variables) @@ -67,7 +63,7 @@ def load_input_data(self) -> FieldSet: fieldset = self._generate_fieldset() except Exception as e: raise CopernicusCatalogueError( - f"Failed to load input data directly from Copernicus Marine (or local data) for instrument '{self.name}'. Original error: {e}" + f"Failed to load input data directly from Copernicus Marine (or local data) for instrument '{self.__class__.__name__}'. Original error: {e}" ) from e # interpolation methods @@ -105,14 +101,14 @@ def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" if not self.verbose_progress: with yaspin( - text=f"Simulating {self.name} measurements... ", + text=f"Simulating {self.__class__.__name__.split('Instrument')[0]} measurements... ", side="right", spinner=ship_spinner, ) as spinner: self.simulate(measurements, out_path) spinner.ok("✅\n") else: - print(f"Simulating {self.name} measurements... ") + print(f"Simulating {self.__class__.__name__} measurements... ") self.simulate(measurements, out_path) print("\n") diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 4684c822..262fc799 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import TYPE_CHECKING, ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType @@ -79,19 +79,12 @@ def _ctd_cast(particle, fieldset, time): class CTDInstrument(Instrument): """CTD instrument class.""" - def __init__(self, expedition, directory, from_data): + def __init__(self, expedition, from_data): """Initialize CTDInstrument.""" - filenames = { - "S": f"{CTD.name}_s.nc", - "T": f"{CTD.name}_t.nc", - } variables = {"S": "so", "T": "thetao"} super().__init__( - CTD.name, expedition, - directory, - filenames, variables, add_bathymetry=True, allow_time_extrapolation=True, diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index d3adcec3..1a9ba612 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -101,17 +101,8 @@ def _ctd_bgc_cast(particle, fieldset, time): class CTD_BGCInstrument(Instrument): """CTD_BGC instrument class.""" - def __init__(self, expedition, directory, from_data): + def __init__(self, expedition, from_data): """Initialize CTD_BGCInstrument.""" - filenames = { - "o2": f"{CTD_BGC.name}_o2.nc", - "chl": f"{CTD_BGC.name}_chl.nc", - "no3": f"{CTD_BGC.name}_no3.nc", - "po4": f"{CTD_BGC.name}_po4.nc", - "ph": f"{CTD_BGC.name}_ph.nc", - "phyc": f"{CTD_BGC.name}_phyc.nc", - "nppv": f"{CTD_BGC.name}_nppv.nc", - } variables = { "o2": "o2", "chl": "chl", @@ -122,10 +113,7 @@ def __init__(self, expedition, directory, from_data): "nppv": "nppv", } super().__init__( - CTD_BGC.name, expedition, - directory, - filenames, variables, add_bathymetry=True, allow_time_extrapolation=True, diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 3ee8ad1c..ddc877b6 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable +from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -63,13 +63,8 @@ def _check_lifetime(particle, fieldset, time): class DrifterInstrument(Instrument): """Drifter instrument class.""" - def __init__(self, expedition, directory, from_data): + def __init__(self, expedition, from_data): """Initialize DrifterInstrument.""" - filenames = { - "U": f"{Drifter.name}_uv.nc", - "V": f"{Drifter.name}_uv.nc", - "T": f"{Drifter.name}_t.nc", - } variables = {"U": "uo", "V": "vo", "T": "thetao"} buffer_spec = { "latlon": 6.0, # [degrees] @@ -81,10 +76,7 @@ def __init__(self, expedition, directory, from_data): } super().__init__( - Drifter.name, expedition, - directory, - filenames, variables, add_bathymetry=False, allow_time_extrapolation=False, diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 6d8d4575..52d2976b 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np -from parcels import ParticleSet, ScipyParticle, Variable +from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_instrument @@ -55,19 +55,12 @@ def _sample_temperature(particle, fieldset, time): class Underwater_STInstrument(Instrument): """Underwater_ST instrument class.""" - def __init__(self, expedition, directory, from_data): + def __init__(self, expedition, from_data): """Initialize Underwater_STInstrument.""" - filenames = { - "S": f"{Underwater_ST.name}_s.nc", - "T": f"{Underwater_ST.name}_t.nc", - } variables = {"S": "so", "T": "thetao"} super().__init__( - Underwater_ST.name, expedition, - directory, - filenames, variables, add_bathymetry=False, allow_time_extrapolation=True, diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index df68122c..ef1339c5 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -77,20 +77,11 @@ def _xbt_cast(particle, fieldset, time): class XBTInstrument(Instrument): """XBT instrument class.""" - def __init__(self, expedition, directory, from_data): + def __init__(self, expedition, from_data): """Initialize XBTInstrument.""" - filenames = { - "U": f"{XBT.name}_uv.nc", - "V": f"{XBT.name}_uv.nc", - "S": f"{XBT.name}_s.nc", - "T": f"{XBT.name}_t.nc", - } variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} super().__init__( - XBT.name, expedition, - directory, - filenames, variables, add_bathymetry=True, allow_time_extrapolation=True, diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py index daa822f0..5ad14c64 100644 --- a/tests/cli/test_run.py +++ b/tests/cli/test_run.py @@ -19,10 +19,9 @@ def _simulate_schedule(projection, expedition): class DummyInstrument: """Dummy instrument class that just creates empty output directories.""" - def __init__(self, expedition, directory, from_data=None): + def __init__(self, expedition, from_data=None): """Initialize DummyInstrument.""" self.expedition = expedition - self.directory = Path(directory) self.from_data = from_data def execute(self, measurements, out_path): diff --git a/tests/instruments/test_adcp.py b/tests/instruments/test_adcp.py index f7802dc7..d87dae42 100644 --- a/tests/instruments/test_adcp.py +++ b/tests/instruments/test_adcp.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.instruments.adcp import ADCPInstrument from virtualship.models import Location, Spacetime @@ -85,10 +85,10 @@ class adcp_config: num_bins = NUM_BINS expedition = DummyExpedition() - directory = tmpdir + from_data = None - adcp_instrument = ADCPInstrument(expedition, directory, from_data) + adcp_instrument = ADCPInstrument(expedition, from_data) out_path = tmpdir.join("out.zarr") adcp_instrument.load_input_data = lambda: fieldset diff --git a/tests/instruments/test_argo_float.py b/tests/instruments/test_argo_float.py index f974e19b..1dfa4060 100644 --- a/tests/instruments/test_argo_float.py +++ b/tests/instruments/test_argo_float.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.instruments.argo_float import ArgoFloat, ArgoFloatInstrument from virtualship.models import Location, Spacetime @@ -58,10 +58,9 @@ class DummyExpedition: pass expedition = DummyExpedition() - directory = tmpdir from_data = None - argo_instrument = ArgoFloatInstrument(expedition, directory, from_data) + argo_instrument = ArgoFloatInstrument(expedition, from_data) out_path = tmpdir.join("out.zarr") argo_instrument.load_input_data = lambda: fieldset diff --git a/tests/instruments/test_base.py b/tests/instruments/test_base.py index 272290e6..6aec1b66 100644 --- a/tests/instruments/test_base.py +++ b/tests/instruments/test_base.py @@ -33,10 +33,7 @@ def test_load_input_data(mock_copernicusmarine, mock_select_product_id, mock_Fie mock_fieldset.__getitem__.side_effect = lambda k: MagicMock() mock_copernicusmarine.open_dataset.return_value = MagicMock() dummy = DummyInstrument( - name="test", expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), - directory="/tmp", - filenames={"A": "a.nc"}, variables={"A": "a"}, add_bathymetry=False, allow_time_extrapolation=False, @@ -51,10 +48,7 @@ def test_load_input_data(mock_copernicusmarine, mock_select_product_id, mock_Fie def test_execute_calls_simulate(monkeypatch): dummy = DummyInstrument( - name="test", expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), - directory="/tmp", - filenames={"A": "a.nc"}, variables={"A": "a"}, add_bathymetry=False, allow_time_extrapolation=False, @@ -68,10 +62,7 @@ def test_execute_calls_simulate(monkeypatch): def test_get_spec_value_buffer_and_limit(): dummy = DummyInstrument( - name="test", expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), - directory="/tmp", - filenames={"A": "a.nc"}, variables={"A": "a"}, add_bathymetry=False, allow_time_extrapolation=False, @@ -87,10 +78,7 @@ def test_get_spec_value_buffer_and_limit(): def test_generate_fieldset_combines_fields(monkeypatch): dummy = DummyInstrument( - name="test", expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), - directory="/tmp", - filenames={"A": "a.nc", "B": "b.nc"}, variables={"A": "a", "B": "b"}, add_bathymetry=False, allow_time_extrapolation=False, @@ -115,10 +103,7 @@ def test_generate_fieldset_combines_fields(monkeypatch): def test_load_input_data_error(monkeypatch): dummy = DummyInstrument( - name="test", expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), - directory="/tmp", - filenames={"A": "a.nc"}, variables={"A": "a"}, add_bathymetry=False, allow_time_extrapolation=False, diff --git a/tests/instruments/test_ctd.py b/tests/instruments/test_ctd.py index cde67a90..0382e159 100644 --- a/tests/instruments/test_ctd.py +++ b/tests/instruments/test_ctd.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr -from parcels import Field, FieldSet +from parcels import Field, FieldSet from virtualship.instruments.ctd import CTD, CTDInstrument from virtualship.models import Location, Spacetime @@ -129,10 +129,10 @@ class space_time_region: )() expedition = DummyExpedition() - directory = tmpdir + from_data = None - ctd_instrument = CTDInstrument(expedition, directory, from_data) + ctd_instrument = CTDInstrument(expedition, from_data) out_path = tmpdir.join("out.zarr") ctd_instrument.load_input_data = lambda: fieldset diff --git a/tests/instruments/test_ctd_bgc.py b/tests/instruments/test_ctd_bgc.py index c5836542..7d1676ed 100644 --- a/tests/instruments/test_ctd_bgc.py +++ b/tests/instruments/test_ctd_bgc.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr -from parcels import Field, FieldSet +from parcels import Field, FieldSet from virtualship.instruments.ctd_bgc import CTD_BGC, CTD_BGCInstrument from virtualship.models import Location, Spacetime @@ -167,10 +167,9 @@ class DummyExpedition: pass expedition = DummyExpedition() - directory = tmpdir from_data = None - ctd_bgc_instrument = CTD_BGCInstrument(expedition, directory, from_data) + ctd_bgc_instrument = CTD_BGCInstrument(expedition, from_data) out_path = tmpdir.join("out.zarr") ctd_bgc_instrument.load_input_data = lambda: fieldset diff --git a/tests/instruments/test_drifter.py b/tests/instruments/test_drifter.py index d160a09f..27bc7ffd 100644 --- a/tests/instruments/test_drifter.py +++ b/tests/instruments/test_drifter.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.instruments.drifter import Drifter, DrifterInstrument from virtualship.models import Location, Spacetime @@ -57,10 +57,10 @@ class DummyExpedition: pass expedition = DummyExpedition() - directory = tmpdir + from_data = None - drifter_instrument = DrifterInstrument(expedition, directory, from_data) + drifter_instrument = DrifterInstrument(expedition, from_data) out_path = tmpdir.join("out.zarr") drifter_instrument.load_input_data = lambda: fieldset diff --git a/tests/instruments/test_ship_underwater_st.py b/tests/instruments/test_ship_underwater_st.py index e5acdd0e..96778f87 100644 --- a/tests/instruments/test_ship_underwater_st.py +++ b/tests/instruments/test_ship_underwater_st.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.instruments.ship_underwater_st import Underwater_STInstrument from virtualship.models import Location, Spacetime @@ -72,10 +72,10 @@ class DummyExpedition: pass expedition = DummyExpedition() - directory = tmpdir + from_data = None - st_instrument = Underwater_STInstrument(expedition, directory, from_data) + st_instrument = Underwater_STInstrument(expedition, from_data) out_path = tmpdir.join("out.zarr") st_instrument.load_input_data = lambda: fieldset diff --git a/tests/instruments/test_xbt.py b/tests/instruments/test_xbt.py index 6c33a6dc..52412510 100644 --- a/tests/instruments/test_xbt.py +++ b/tests/instruments/test_xbt.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr -from parcels import Field, FieldSet +from parcels import Field, FieldSet from virtualship.instruments.xbt import XBT, XBTInstrument from virtualship.models import Location, Spacetime @@ -100,10 +100,10 @@ class DummyExpedition: pass expedition = DummyExpedition() - directory = tmpdir + from_data = None - xbt_instrument = XBTInstrument(expedition, directory, from_data) + xbt_instrument = XBTInstrument(expedition, from_data) out_path = tmpdir.join("out.zarr") xbt_instrument.load_input_data = lambda: fieldset From 94d5be28f87c59428c8860ef9795411f7786dde1 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 10:04:00 +0100 Subject: [PATCH 86/97] change variable name --- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 4 ++-- src/virtualship/instruments/base.py | 12 ++++++------ src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 4 ++-- src/virtualship/instruments/ship_underwater_st.py | 2 +- src/virtualship/instruments/xbt.py | 2 +- tests/instruments/test_base.py | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 46f34faf..0270b3e6 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -64,7 +64,7 @@ def __init__(self, expedition, from_data): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, - buffer_spec=None, + spacetime_buffer_size=None, limit_spec=None, from_data=from_data, ) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 69095da9..2253c1a1 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -140,7 +140,7 @@ class ArgoFloatInstrument(Instrument): def __init__(self, expedition, from_data): """Initialize ArgoFloatInstrument.""" variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} - buffer_spec = { + spacetime_buffer_size = { "latlon": 3.0, # [degrees] "time": 21.0, # [days] } @@ -151,7 +151,7 @@ def __init__(self, expedition, from_data): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, - buffer_spec=buffer_spec, + spacetime_buffer_size=spacetime_buffer_size, limit_spec=None, from_data=from_data, ) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index d9339942..e837ba79 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -37,7 +37,7 @@ def __init__( allow_time_extrapolation: bool, verbose_progress: bool, from_data: Path | None, - buffer_spec: dict | None = None, + spacetime_buffer_size: dict | None = None, limit_spec: dict | None = None, ): """Initialise instrument.""" @@ -54,7 +54,7 @@ def __init__( self.add_bathymetry = add_bathymetry self.allow_time_extrapolation = allow_time_extrapolation self.verbose_progress = verbose_progress - self.buffer_spec = buffer_spec + self.spacetime_buffer_size = spacetime_buffer_size self.limit_spec = limit_spec def load_input_data(self) -> FieldSet: @@ -78,8 +78,8 @@ def load_input_data(self) -> FieldSet: if self.add_bathymetry: bathymetry_field = _get_bathy_data( self.expedition.schedule.space_time_region, - latlon_buffer=self.buffer_spec.get("latlon") - if self.buffer_spec + latlon_buffer=self.spacetime_buffer_size.get("latlon") + if self.spacetime_buffer_size else None, from_data=self.from_data, ).bathymetry @@ -209,8 +209,8 @@ def _generate_fieldset(self) -> FieldSet: return base_fieldset def _get_spec_value(self, spec_type: str, key: str, default=None): - """Helper to extract a value from buffer_spec or limit_spec.""" - spec = self.buffer_spec if spec_type == "buffer" else self.limit_spec + """Helper to extract a value from spacetime_buffer_size or limit_spec.""" + spec = self.spacetime_buffer_size if spec_type == "buffer" else self.limit_spec return spec.get(key) if spec and spec.get(key) is not None else default def _find_files_in_timerange( diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 262fc799..f90c5fc0 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -89,7 +89,7 @@ def __init__(self, expedition, from_data): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, - buffer_spec=None, + spacetime_buffer_size=None, limit_spec=None, from_data=from_data, ) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 1a9ba612..039e850c 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -118,7 +118,7 @@ def __init__(self, expedition, from_data): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, - buffer_spec=None, + spacetime_buffer_size=None, limit_spec=None, from_data=from_data, ) diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index ddc877b6..8428ad7c 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -66,7 +66,7 @@ class DrifterInstrument(Instrument): def __init__(self, expedition, from_data): """Initialize DrifterInstrument.""" variables = {"U": "uo", "V": "vo", "T": "thetao"} - buffer_spec = { + spacetime_buffer_size = { "latlon": 6.0, # [degrees] "time": 21.0, # [days] } @@ -81,7 +81,7 @@ def __init__(self, expedition, from_data): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, - buffer_spec=buffer_spec, + spacetime_buffer_size=spacetime_buffer_size, limit_spec=limit_spec, from_data=from_data, ) diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 52d2976b..e6af25b3 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -65,7 +65,7 @@ def __init__(self, expedition, from_data): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, - buffer_spec=None, + spacetime_buffer_size=None, limit_spec=None, from_data=from_data, ) diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index ef1339c5..3e11b55d 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -86,7 +86,7 @@ def __init__(self, expedition, from_data): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, - buffer_spec=None, + spacetime_buffer_size=None, limit_spec=None, from_data=from_data, ) diff --git a/tests/instruments/test_base.py b/tests/instruments/test_base.py index 6aec1b66..29d319ba 100644 --- a/tests/instruments/test_base.py +++ b/tests/instruments/test_base.py @@ -67,7 +67,7 @@ def test_get_spec_value_buffer_and_limit(): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=False, - buffer_spec={"latlon": 5.0}, + spacetime_buffer_size={"latlon": 5.0}, limit_spec={"depth_min": 10.0}, from_data=None, ) From 890cd94c2cae684971bc8ab1503b9cb4ac4f8b26 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 10:13:36 +0100 Subject: [PATCH 87/97] make _find_files_in_timerange standalone from Instrument base class --- src/virtualship/instruments/base.py | 79 +++-------------------------- src/virtualship/utils.py | 74 ++++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 75 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index e837ba79..5faf9a76 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,10 +1,8 @@ from __future__ import annotations import abc -import glob -import re from collections import OrderedDict -from datetime import datetime, timedelta +from datetime import timedelta from pathlib import Path from typing import TYPE_CHECKING @@ -16,6 +14,7 @@ from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, + _find_files_in_timerange, _find_nc_file_with_variable, _get_bathy_data, _select_product_id, @@ -108,7 +107,9 @@ def execute(self, measurements: list, out_path: str | Path) -> None: self.simulate(measurements, out_path) spinner.ok("✅\n") else: - print(f"Simulating {self.__class__.__name__} measurements... ") + print( + f"Simulating {self.__class__.__name__.split('Instrument')[0]} measurements... " + ) self.simulate(measurements, out_path) print("\n") @@ -174,7 +175,7 @@ def _generate_fieldset(self) -> FieldSet: self.expedition.schedule.space_time_region.time_range.end_time ) - files = self._find_files_in_timerange( + files = _find_files_in_timerange( data_dir, schedule_start, schedule_end, @@ -212,71 +213,3 @@ def _get_spec_value(self, spec_type: str, key: str, default=None): """Helper to extract a value from spacetime_buffer_size or limit_spec.""" spec = self.spacetime_buffer_size if spec_type == "buffer" else self.limit_spec return spec.get(key) if spec and spec.get(key) is not None else default - - def _find_files_in_timerange( - self, - data_dir: Path, - schedule_start, - schedule_end, - date_pattern=r"\d{4}_\d{2}_\d{2}", - date_fmt="%Y_%m_%d", - ) -> list: - """Find all files in data_dir whose filenames contain a date within [schedule_start, schedule_end] (inclusive).""" - # TODO: scope to make this more flexible for different date patterns / formats ... ? - - all_files = glob.glob(str(data_dir.joinpath("*"))) - if not all_files: - raise ValueError( - f"No files found in data directory {data_dir}. Please ensure the directory contains files with 'P1D' or 'P1M' in their names as per Copernicus Marine Product ID naming conventions." - ) - - if all("P1D" in s for s in all_files): - t_resolution = "daily" - elif all("P1M" in s for s in all_files): - t_resolution = "monthly" - else: - raise ValueError( - f"Could not determine time resolution from filenames in data directory. Please ensure all filenames in {data_dir} contain either 'P1D' (daily) or 'P1M' (monthly), " - f"as per the Copernicus Marine Product ID naming conventions." - ) - - if t_resolution == "monthly": - t_min = schedule_start.date().replace( - day=1 - ) # first day of month of the schedule start date - t_max = ( - schedule_end.date() - + timedelta( - days=32 - ) # buffer to ensure fieldset end date is always longer than schedule end date for monthly data - ) - else: # daily - t_min = schedule_start.date() - t_max = schedule_end.date() - - files_with_dates = [] - for file in data_dir.iterdir(): - if file.is_file(): - match = re.search(date_pattern, file.name) - if match: - file_date = datetime.strptime( - match.group(), date_fmt - ).date() # normalise to date only for comparison (given start/end dates have hour/minute components which may exceed those in file_date) - if t_min <= file_date <= t_max: - files_with_dates.append((file_date, file.name)) - - files_with_dates.sort( - key=lambda x: x[0] - ) # sort by extracted date; more robust than relying on filesystem order - - # catch if not enough data coverage found for the requested time range - if files_with_dates[-1][0] < schedule_end.date(): - raise ValueError( - f"Not enough data coverage found in {data_dir} for the requested time range {schedule_start} to {schedule_end}. " - f"Latest available data is for date {files_with_dates[-1][0]}." - f"If using monthly data, please ensure that the last month downloaded covers the schedule end date + 1 month." - f"See documentation for more details: <>" - # TODO: add link to relevant documentation! - ) - - return [fname for _, fname in files_with_dates] diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index a8f61b49..ce86b1de 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -1,8 +1,10 @@ from __future__ import annotations +import glob import os +import re import warnings -from datetime import timedelta +from datetime import datetime, timedelta from functools import lru_cache from importlib.resources import files from pathlib import Path @@ -11,8 +13,8 @@ import copernicusmarine import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: @@ -491,3 +493,71 @@ def _find_nc_file_with_variable(data_dir: Path, var: str) -> str | None: except Exception: continue return None + + +def _find_files_in_timerange( + data_dir: Path, + schedule_start, + schedule_end, + date_pattern=r"\d{4}_\d{2}_\d{2}", + date_fmt="%Y_%m_%d", +) -> list: + """Find all files in data_dir whose filenames contain a date within [schedule_start, schedule_end] (inclusive).""" + # TODO: scope to make this more flexible for different date patterns / formats ... ? + + all_files = glob.glob(str(data_dir.joinpath("*"))) + if not all_files: + raise ValueError( + f"No files found in data directory {data_dir}. Please ensure the directory contains files with 'P1D' or 'P1M' in their names as per Copernicus Marine Product ID naming conventions." + ) + + if all("P1D" in s for s in all_files): + t_resolution = "daily" + elif all("P1M" in s for s in all_files): + t_resolution = "monthly" + else: + raise ValueError( + f"Could not determine time resolution from filenames in data directory. Please ensure all filenames in {data_dir} contain either 'P1D' (daily) or 'P1M' (monthly), " + f"as per the Copernicus Marine Product ID naming conventions." + ) + + if t_resolution == "monthly": + t_min = schedule_start.date().replace( + day=1 + ) # first day of month of the schedule start date + t_max = ( + schedule_end.date() + + timedelta( + days=32 + ) # buffer to ensure fieldset end date is always longer than schedule end date for monthly data + ) + else: # daily + t_min = schedule_start.date() + t_max = schedule_end.date() + + files_with_dates = [] + for file in data_dir.iterdir(): + if file.is_file(): + match = re.search(date_pattern, file.name) + if match: + file_date = datetime.strptime( + match.group(), date_fmt + ).date() # normalise to date only for comparison (given start/end dates have hour/minute components which may exceed those in file_date) + if t_min <= file_date <= t_max: + files_with_dates.append((file_date, file.name)) + + files_with_dates.sort( + key=lambda x: x[0] + ) # sort by extracted date; more robust than relying on filesystem order + + # catch if not enough data coverage found for the requested time range + if files_with_dates[-1][0] < schedule_end.date(): + raise ValueError( + f"Not enough data coverage found in {data_dir} for the requested time range {schedule_start} to {schedule_end}. " + f"Latest available data is for date {files_with_dates[-1][0]}." + f"If using monthly data, please ensure that the last month downloaded covers the schedule end date + 1 month." + f"See documentation for more details: <>" + # TODO: add link to relevant documentation! + ) + + return [fname for _, fname in files_with_dates] From c0ba080378031c831a688c442bebfa078f84a266 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 10:21:25 +0100 Subject: [PATCH 88/97] Update error docstring --- src/virtualship/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/virtualship/errors.py b/src/virtualship/errors.py index fad5863d..ac1aa8a1 100644 --- a/src/virtualship/errors.py +++ b/src/virtualship/errors.py @@ -23,7 +23,7 @@ class ScheduleError(RuntimeError): class InstrumentsConfigError(RuntimeError): - """An error in the config.""" + """An error in the InstrumentsConfig.""" pass From 141530a72c2f115f39caefcbaf3fda4dcd40ad88 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 11:28:48 +0100 Subject: [PATCH 89/97] update plan UI logic to update space-time region dynamically --- src/virtualship/cli/_plan.py | 61 ++++++++++++++++++++++++++++++------ src/virtualship/utils.py | 5 ++- tests/test_utils.py | 8 ++--- 3 files changed, 58 insertions(+), 16 deletions(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index a164cba3..8b2adb15 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -590,14 +590,6 @@ def _update_instrument_configs(self): ) def _update_schedule(self): - spatial_range = SpatialRange( - minimum_longitude=self.query_one("#min_lon").value, - maximum_longitude=self.query_one("#max_lon").value, - minimum_latitude=self.query_one("#min_lat").value, - maximum_latitude=self.query_one("#max_lat").value, - minimum_depth=self.query_one("#min_depth").value, - maximum_depth=self.query_one("#max_depth").value, - ) start_time_input = self.query_one("#start_time").value end_time_input = self.query_one("#end_time").value waypoint_times = [ @@ -614,8 +606,8 @@ def _update_schedule(self): else: end_time = end_time_input time_range = TimeRange(start_time=start_time, end_time=end_time) - self.expedition.schedule.space_time_region.spatial_range = spatial_range self.expedition.schedule.space_time_region.time_range = time_range + for i, wp in enumerate(self.expedition.schedule.waypoints): wp.location = Location( latitude=float(self.query_one(f"#wp{i}_lat").value), @@ -640,6 +632,57 @@ def _update_schedule(self): elif switch_on: wp.instrument.append(instrument) + # take min/max lat/lon to be most extreme values of waypoints or space_time_region inputs (so as to cover possibility of user edits in either place) + # also prevents situation where e.g. user defines a space time region inconsistent with waypoint locations and vice versa (warning also provided) + waypoint_lats = [ + wp.location.latitude for wp in self.expedition.schedule.waypoints + ] + waypoint_lons = [ + wp.location.longitude for wp in self.expedition.schedule.waypoints + ] + wp_min_lat, wp_max_lat = ( + min(waypoint_lats) if waypoint_lats else -90.0, + max(waypoint_lats) if waypoint_lats else 90.0, + ) + wp_min_lon, wp_max_lon = ( + min(waypoint_lons) if waypoint_lons else -180.0, + max(waypoint_lons) if waypoint_lons else 180.0, + ) + + st_reg_min_lat = float(self.query_one("#min_lat").value) + st_reg_max_lat = float(self.query_one("#max_lat").value) + st_reg_min_lon = float(self.query_one("#min_lon").value) + st_reg_max_lon = float(self.query_one("#max_lon").value) + + min_lat = min(wp_min_lat, st_reg_min_lat) + max_lat = max(wp_max_lat, st_reg_max_lat) + min_lon = min(wp_min_lon, st_reg_min_lon) + max_lon = max(wp_max_lon, st_reg_max_lon) + + spatial_range = SpatialRange( + minimum_longitude=min_lon, + maximum_longitude=max_lon, + minimum_latitude=min_lat, + maximum_latitude=max_lat, + minimum_depth=self.query_one("#min_depth").value, + maximum_depth=self.query_one("#max_depth").value, + ) + self.expedition.schedule.space_time_region.spatial_range = spatial_range + + # provide warning if user defines a space time region inconsistent with waypoint locations + if ( + (wp_min_lat < st_reg_min_lat) + or (wp_max_lat > st_reg_max_lat) + or (wp_min_lon < st_reg_min_lon) + or (wp_max_lon > st_reg_max_lon) + ): + self.notify( + "[b]WARNING[/b]. One or more waypoint locations lie outside the defined space-time region. Take care if manually adjusting the space-time region." + "\n\nThe space-time region will be automatically adjusted on saving to include all waypoint locations.", + severity="warning", + timeout=10, + ) + @on(Input.Changed) def show_invalid_reasons(self, event: Input.Changed) -> None: input_id = event.input.id diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index ce86b1de..6dc25cde 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -427,7 +427,7 @@ def _get_bathy_data( except Exception as e: # TODO: link to documentation on expected data structure!! raise RuntimeError( - f"\n\n❗️ Could not find bathymetry variable '{var}' in data directory '{from_data}/bathymetry/'.\n\n❗️ Is the pre-downloaded data directory structure compliant with VirtualShip expectations?\n\n❗️ See for more information on expectations: <<>>\n" + f"\n\n❗️ Could not find bathymetry variable '{var}' in data directory '{from_data}/bathymetry/'.\n\n❗️ Is the pre-downloaded data directory structure compliant with VirtualShip expectations?\n\n❗️ See the docs for more information on expectations: https://virtualship.readthedocs.io/en/latest/user-guide/index.html#documentation\n" ) from e ds_bathymetry = xr.open_dataset(bathy_dir.joinpath(filename)) bathymetry_variables = {"bathymetry": "deptho"} @@ -556,8 +556,7 @@ def _find_files_in_timerange( f"Not enough data coverage found in {data_dir} for the requested time range {schedule_start} to {schedule_end}. " f"Latest available data is for date {files_with_dates[-1][0]}." f"If using monthly data, please ensure that the last month downloaded covers the schedule end date + 1 month." - f"See documentation for more details: <>" - # TODO: add link to relevant documentation! + f"See the docs for more details: https://virtualship.readthedocs.io/en/latest/user-guide/index.html#documentation" ) return [fname for _, fname in files_with_dates] diff --git a/tests/test_utils.py b/tests/test_utils.py index 3018411b..6004ea69 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,9 +3,9 @@ import numpy as np import pytest import xarray as xr -from parcels import FieldSet import virtualship.utils +from parcels import FieldSet from virtualship.models.expedition import Expedition from virtualship.utils import ( _find_nc_file_with_variable, @@ -251,14 +251,14 @@ def test_data_dir_and_filename_compliance(): ) # Check for date_pattern in _find_files_in_timerange - assert 'date_pattern=r"\\d{4}_\\d{2}_\\d{2}"' in base_code, ( + assert 'date_pattern=r"\\d{4}_\\d{2}_\\d{2}"' in utils_code, ( "Expected date_pattern r'\\d{4}_\\d{2}_\\d{2}' not found in _find_files_in_timerange. This indicates a drift between docs and implementation." ) # Check for P1D and P1M in t_resolution logic - assert 'if all("P1D" in s for s in all_files):' in base_code, ( + assert 'if all("P1D" in s for s in all_files):' in utils_code, ( "Expected check for 'P1D' in all_files not found in _find_files_in_timerange. This indicates a drift between docs and implementation." ) - assert 'elif all("P1M" in s for s in all_files):' in base_code, ( + assert 'elif all("P1M" in s for s in all_files):' in utils_code, ( "Expected check for 'P1M' in all_files not found in _find_files_in_timerange. This indicates a drift between docs and implementation." ) From 9cb27a7c7e452535dd1b9357d06779a0fda3d3f1 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 14:38:38 +0100 Subject: [PATCH 90/97] fix xbt bug --- src/virtualship/instruments/xbt.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 3e11b55d..0d6f6276 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -8,7 +8,7 @@ from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.utils import add_dummy_UV, register_instrument # ===================================================== # SECTION: Dataclass @@ -79,7 +79,7 @@ class XBTInstrument(Instrument): def __init__(self, expedition, from_data): """Initialize XBTInstrument.""" - variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} + variables = {"T": "thetao"} super().__init__( expedition, variables, @@ -94,7 +94,7 @@ def __init__(self, expedition, from_data): def simulate(self, measurements, out_path) -> None: """Simulate XBT measurements.""" DT = 10.0 # dt of XBT simulation integrator - OUTPUT_DT = timedelta(seconds=1) + OUTPUT_DT = timedelta(seconds=10) if len(measurements) == 0: print( @@ -105,8 +105,15 @@ def simulate(self, measurements, out_path) -> None: fieldset = self.load_input_data() - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + # add dummy U + add_dummy_UV(fieldset) # TODO: parcels v3 bodge; remove when parcels v4 is used + + fieldset_starttime = fieldset.T.grid.time_origin.fulltime( + fieldset.T.grid.time_full[0] + ) + fieldset_endtime = fieldset.T.grid.time_origin.fulltime( + fieldset.T.grid.time_full[-1] + ) # deploy time for all xbts should be later than fieldset start time if not all( From 6b6d39509d1baab777527e997bd09bf861ba703d Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:07:59 +0100 Subject: [PATCH 91/97] add warnings to ADCP max depth config if exceeds authentic limits --- src/virtualship/instruments/adcp.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 0270b3e6..22f9859b 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -71,7 +71,18 @@ def __init__(self, expedition, from_data): def simulate(self, measurements, out_path) -> None: """Simulate ADCP measurements.""" - MAX_DEPTH = self.expedition.instruments_config.adcp_config.max_depth_meter + config_max_depth = ( + self.expedition.instruments_config.adcp_config.max_depth_meter + ) + + if config_max_depth < -1600.0: + print( + f"\n\n⚠️ Warning: The configured ADCP max depth of {abs(config_max_depth)} m exceeds the 1600 m limit for the technology (e.g. https://www.geomar.de/en/research/fb1/fb1-po/observing-systems/adcp)." + "\n\n This expedition will continue using the prescribed configuration. However, note, the results will not necessarily represent authentic ADCP instrument readings and could also lead to slower simulations ." + "\n\n If this was unintented, consider re-adjusting your ADCP configuration in your expedition.yaml or via `virtualship plan`.\n\n" + ) + + MAX_DEPTH = config_max_depth MIN_DEPTH = -5.0 NUM_BINS = self.expedition.instruments_config.adcp_config.num_bins From b26636bfb62052998ec9a5d08e2e33eacbc58a0f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 20 Nov 2025 14:11:18 +0000 Subject: [PATCH 92/97] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/instruments/base.py | 2 +- src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 2 +- src/virtualship/instruments/ship_underwater_st.py | 2 +- src/virtualship/instruments/xbt.py | 2 +- src/virtualship/utils.py | 2 +- tests/instruments/test_adcp.py | 2 +- tests/instruments/test_argo_float.py | 2 +- tests/instruments/test_ctd.py | 2 +- tests/instruments/test_ctd_bgc.py | 2 +- tests/instruments/test_drifter.py | 2 +- tests/instruments/test_ship_underwater_st.py | 2 +- tests/instruments/test_xbt.py | 2 +- tests/test_utils.py | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 22f9859b..2a761e14 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 2253c1a1..204f0b3d 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,7 +4,6 @@ from typing import ClassVar import numpy as np - from parcels import ( AdvectionRK4, JITParticle, @@ -12,6 +11,7 @@ StatusCode, Variable, ) + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 5faf9a76..22b0b54a 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -8,9 +8,9 @@ import copernicusmarine import xarray as xr +from parcels import FieldSet from yaspin import yaspin -from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index f90c5fc0..73248cf9 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import TYPE_CHECKING, ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 039e850c..fab9e07b 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 8428ad7c..e962278d 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index e6af25b3..088a439f 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_instrument diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 0d6f6276..f0f5d130 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 6dc25cde..7e37617b 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -13,8 +13,8 @@ import copernicusmarine import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: diff --git a/tests/instruments/test_adcp.py b/tests/instruments/test_adcp.py index d87dae42..a2a5418a 100644 --- a/tests/instruments/test_adcp.py +++ b/tests/instruments/test_adcp.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.adcp import ADCPInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_argo_float.py b/tests/instruments/test_argo_float.py index 1dfa4060..cbe25d76 100644 --- a/tests/instruments/test_argo_float.py +++ b/tests/instruments/test_argo_float.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.argo_float import ArgoFloat, ArgoFloatInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_ctd.py b/tests/instruments/test_ctd.py index 0382e159..fff5fc4f 100644 --- a/tests/instruments/test_ctd.py +++ b/tests/instruments/test_ctd.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr - from parcels import Field, FieldSet + from virtualship.instruments.ctd import CTD, CTDInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_ctd_bgc.py b/tests/instruments/test_ctd_bgc.py index 7d1676ed..00f30077 100644 --- a/tests/instruments/test_ctd_bgc.py +++ b/tests/instruments/test_ctd_bgc.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr - from parcels import Field, FieldSet + from virtualship.instruments.ctd_bgc import CTD_BGC, CTD_BGCInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_drifter.py b/tests/instruments/test_drifter.py index 27bc7ffd..9253c1a8 100644 --- a/tests/instruments/test_drifter.py +++ b/tests/instruments/test_drifter.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.drifter import Drifter, DrifterInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_ship_underwater_st.py b/tests/instruments/test_ship_underwater_st.py index 96778f87..e7ca18d1 100644 --- a/tests/instruments/test_ship_underwater_st.py +++ b/tests/instruments/test_ship_underwater_st.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.ship_underwater_st import Underwater_STInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_xbt.py b/tests/instruments/test_xbt.py index 52412510..d218025a 100644 --- a/tests/instruments/test_xbt.py +++ b/tests/instruments/test_xbt.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr - from parcels import Field, FieldSet + from virtualship.instruments.xbt import XBT, XBTInstrument from virtualship.models import Location, Spacetime diff --git a/tests/test_utils.py b/tests/test_utils.py index 6004ea69..8bd2338e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,9 +3,9 @@ import numpy as np import pytest import xarray as xr +from parcels import FieldSet import virtualship.utils -from parcels import FieldSet from virtualship.models.expedition import Expedition from virtualship.utils import ( _find_nc_file_with_variable, From ac02bdcf9951b3fb5c557fbee896a10c6f97d542 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:42:16 +0100 Subject: [PATCH 93/97] error messaging for case where measurements cause schedule to be missed --- src/virtualship/expedition/simulate_schedule.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 1d43dc13..06e0c2fc 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -8,6 +8,7 @@ import pyproj +from virtualship.errors import ScheduleError from virtualship.instruments.argo_float import ArgoFloat from virtualship.instruments.ctd import CTD from virtualship.instruments.ctd_bgc import CTD_BGC @@ -121,8 +122,9 @@ def simulate(self) -> ScheduleOk | ScheduleProblem: # check if waypoint was reached in time if waypoint.time is not None and self._time > waypoint.time: - print( + raise ScheduleError( f"Waypoint {wp_i + 1} could not be reached in time. Current time: {self._time}. Waypoint time: {waypoint.time}." + "\n\nHave you ensured that your schedule includes sufficient time for taking measurements such as CTD casts (in addition to the time it takes to sail between waypoints)?\n\n" ) return ScheduleProblem(self._time, wp_i) else: From 7b25526079b1c22b9bfb3c039034bbf2cc6e57b4 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 21 Nov 2025 09:02:24 +0100 Subject: [PATCH 94/97] revert to using ScheduleProblem class --- src/virtualship/expedition/simulate_schedule.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 06e0c2fc..0513de65 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -8,7 +8,6 @@ import pyproj -from virtualship.errors import ScheduleError from virtualship.instruments.argo_float import ArgoFloat from virtualship.instruments.ctd import CTD from virtualship.instruments.ctd_bgc import CTD_BGC @@ -122,7 +121,7 @@ def simulate(self) -> ScheduleOk | ScheduleProblem: # check if waypoint was reached in time if waypoint.time is not None and self._time > waypoint.time: - raise ScheduleError( + print( f"Waypoint {wp_i + 1} could not be reached in time. Current time: {self._time}. Waypoint time: {waypoint.time}." "\n\nHave you ensured that your schedule includes sufficient time for taking measurements such as CTD casts (in addition to the time it takes to sail between waypoints)?\n\n" ) From 28dbe3397f3a3b7b5edf049fbcbd178a5ec82db6 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 21 Nov 2025 09:11:42 +0100 Subject: [PATCH 95/97] add more informative messaging on ScheduleProblem --- src/virtualship/cli/_run.py | 2 +- src/virtualship/expedition/simulate_schedule.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index 6ae7a1cd..f07fbab2 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -98,7 +98,7 @@ def _run(expedition_dir: str | Path, from_data: Path | None = None) -> None: ) if isinstance(schedule_results, ScheduleProblem): print( - "Update your schedule and continue the expedition by running the tool again." + f"SIMULATION PAUSED: update your schedule (`virtualship plan`) and continue the expedition by executing the `virtualship run` command again.\nCheckpoint has been saved to {expedition_dir.joinpath(CHECKPOINT)}." ) _save_checkpoint( Checkpoint( diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 0513de65..0a567b1c 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -123,7 +123,8 @@ def simulate(self) -> ScheduleOk | ScheduleProblem: if waypoint.time is not None and self._time > waypoint.time: print( f"Waypoint {wp_i + 1} could not be reached in time. Current time: {self._time}. Waypoint time: {waypoint.time}." - "\n\nHave you ensured that your schedule includes sufficient time for taking measurements such as CTD casts (in addition to the time it takes to sail between waypoints)?\n\n" + "\n\nHave you ensured that your schedule includes sufficient time for taking measurements, e.g. CTD casts (in addition to the time it takes to sail between waypoints)?\n" + "**Note**, the `virtualship plan` tool will not account for measurement times when verifying the schedule, only the time it takes to sail between waypoints.\n" ) return ScheduleProblem(self._time, wp_i) else: From 9bf6446c705a87d8cf60519bdc831585339d850b Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 21 Nov 2025 09:49:50 +0100 Subject: [PATCH 96/97] change test to mock using data from disk to avoid copernicus calls --- tests/cli/test_run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py index 5ad14c64..e2930dba 100644 --- a/tests/cli/test_run.py +++ b/tests/cli/test_run.py @@ -50,10 +50,13 @@ def test_run(tmp_path, monkeypatch): "virtualship.cli._run.get_instrument_class", lambda itype: DummyInstrument ) - fake_data_dir = None + fake_data_dir = tmp_path / "fake_data" + fake_data_dir.mkdir() _run(expedition_dir, from_data=fake_data_dir) + breakpoint() + results_dir = expedition_dir / "results" assert results_dir.exists() and results_dir.is_dir() From 0d5c5eb0c49049a22d713283e4fd89a52f95631f Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 21 Nov 2025 09:54:04 +0100 Subject: [PATCH 97/97] remove accidental breakpoint --- tests/cli/test_run.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py index e2930dba..19044234 100644 --- a/tests/cli/test_run.py +++ b/tests/cli/test_run.py @@ -55,8 +55,6 @@ def test_run(tmp_path, monkeypatch): _run(expedition_dir, from_data=fake_data_dir) - breakpoint() - results_dir = expedition_dir / "results" assert results_dir.exists() and results_dir.is_dir()