diff --git a/workflow/log_utils.py b/workflow/log_utils.py index 5735d47..67f22db 100644 --- a/workflow/log_utils.py +++ b/workflow/log_utils.py @@ -1,8 +1,8 @@ r"""Structured Logging Utilities for `logging`. -This module support for different structured-logging formats (JSON and +This module supports different structured-logging formats (JSON and text) and decorators for logging function calls and command -executions. The logging functions can capture and structure log +executions. The logging functions capture and structure log messages along with additional contextual information such as function arguments, execution status, and timestamps. @@ -13,12 +13,12 @@ >>> def foo(a, b): >>> return a + b >>> foo(1, 2) -2024-09-18 22:00:51.498268+00:00 INFO example MainThread called function=foo id=... a=1 b=2 -2024-09-18 22:00:51.498268+00:00 INFO example MainThread completed function=foo id=... result=3 +{"timestamp": "2025-02-25T22:00:51.498268Z", "level": "info", "logger": "example", "thread": "MainThread", "event": "called", "function": "foo", "id": "...", "a": 1, "b": 2} +{"timestamp": "2025-02-25T22:00:51.498268Z", "level": "info", "logger": "example", "thread": "MainThread", "event": "completed", "function": "foo", "id": "...", "result": 3} 3 >>> logger = get_logger('example') ->>> logger.info(structured_log('hello world', counter=1)) -2024-09-18 22:00:51.498268+00:00 INFO example MainThread hello world counter=1 +>>> logger.info('hello world', counter=1) +{"timestamp": "2025-02-25T22:00:51.498268Z", "level": "info", "logger": "example", "thread": "MainThread", "event": "hello world", "counter": 1} """ import functools diff --git a/workflow/scripts/bb_sim.py b/workflow/scripts/bb_sim.py index 290b156..ee73a81 100644 --- a/workflow/scripts/bb_sim.py +++ b/workflow/scripts/bb_sim.py @@ -125,14 +125,10 @@ def bb_simulate_station( logger = log_utils.get_logger(__name__) if np.isnan(lf_acc).any(): - logger.error( - log_utils.structured_log("Station LF had NaN waveform", station=station) - ) + logger.error("Station LF had NaN waveform", station=station) raise ValueError(f"Station {station_name} had NaN waveform") if np.isnan(hf_acc).any(): - logger.error( - log_utils.structured_log("Station HF had NaN waveform", station=station) - ) + logger.error("Station HF had NaN waveform", station=station) raise ValueError(f"Station {station_name} had NaN waveform") pga = np.max(np.abs(hf_acc), axis=0) / 981.0 @@ -181,9 +177,13 @@ def bb_simulate_station( def combine_hf_and_lf( realisation_ffp: Annotated[Path, typer.Argument(dir_okay=False, exists=True)], station_vs30_ffp: Annotated[Path, typer.Argument(dir_okay=False, exists=True)], - low_frequency_waveform_directory: Annotated[Path, typer.Argument(file_okay=False, exists=True)], + low_frequency_waveform_directory: Annotated[ + Path, typer.Argument(file_okay=False, exists=True) + ], high_frequency_waveform_file: Annotated[Path, typer.Argument(exists=True)], - velocity_model_directory: Annotated[Path, typer.Argument(file_okay=False, exists=True)], + velocity_model_directory: Annotated[ + Path, typer.Argument(file_okay=False, exists=True) + ], output_ffp: Annotated[Path, typer.Argument(dir_okay=False, writable=True)], ): """Combine low-frequency and high-frequency seismic waveforms. diff --git a/workflow/scripts/check_domain.py b/workflow/scripts/check_domain.py index 8e1537c..91b7608 100644 --- a/workflow/scripts/check_domain.py +++ b/workflow/scripts/check_domain.py @@ -26,6 +26,7 @@ ------------- See the output of `check-domain --help`. """ + from pathlib import Path from typing import Annotated @@ -76,10 +77,8 @@ def check_domain( hausdorf_distance = shapely.hausdorff_distance(srf_geometry, domain.domain.polygon) if hausdorf_distance < 1000: logger.warning( - log_utils.structured_log( - "SRF geometry is close the edge of the domain. This could indicate a patch outside the domain, but may not be an error in its own right.", - closest_distance=hausdorf_distance, - ), + "SRF geometry is close the edge of the domain. This could indicate a patch outside the domain, but may not be an error in its own right.", + closest_distance=hausdorf_distance, ) raise typer.Exit(code=1) @@ -95,9 +94,7 @@ def check_domain( ] ): logger.error( - log_utils.structured_log( - "The expected velocity model size does not match the computed velocity model size", - expected_size=velocity_model_estimated_size, - ), + "The expected velocity model size does not match the computed velocity model size", + expected_size=velocity_model_estimated_size, ) raise typer.Exit(code=1) diff --git a/workflow/scripts/check_srf.py b/workflow/scripts/check_srf.py index 9265274..2397e2d 100644 --- a/workflow/scripts/check_srf.py +++ b/workflow/scripts/check_srf.py @@ -100,37 +100,27 @@ def check_srf( ) if not np.isclose(srf_magnitude, magnitude, atol=5e-3): logger.error( - log_utils.structured_log( - "Mismatch SRF magnitude", - srf_magnitude=srf_magnitude, - realisation_magnitude=magnitude, - ) + "Mismatch SRF magnitude", + srf_magnitude=srf_magnitude, + realisation_magnitude=magnitude, ) raise typer.Exit(code=1) except RealisationParseError: pass if srf_magnitude >= 11: - logger.error( - log_utils.structured_log( - "Implausible SRF magnitude", srf_magnitude=magnitude - ) - ) + logger.error("Implausible SRF magnitude", srf_magnitude=magnitude) raise typer.Exit(code=1) if (srf_file.points["dep"] < 0).any(): logger.error( - log_utils.structured_log( - "Negative SRF depth detected", min_depth=srf_file.points["depth"].min() - ) + "Negative SRF depth detected", min_depth=srf_file.points["depth"].min() ) raise typer.Exit(code=1) if not np.isclose(srf_file.points["tinit"].min(), 0): logger.warning( - log_utils.structured_log( - "SRF does not begin at zero (this may not be an error depending on SRF setup)", - tinit=srf_file.points["tinit"].min(), - ) + "SRF does not begin at zero (this may not be an error depending on SRF setup)", + tinit=srf_file.points["tinit"].min(), ) raise typer.Exit(code=1) diff --git a/workflow/scripts/generate_velocity_model_parameters.py b/workflow/scripts/generate_velocity_model_parameters.py index 90e398f..279772e 100644 --- a/workflow/scripts/generate_velocity_model_parameters.py +++ b/workflow/scripts/generate_velocity_model_parameters.py @@ -84,7 +84,6 @@ def get_nz_outline_polygon() -> Polygon: return shapely.union(south_island, north_island) - def estimate_simulation_duration( bounding_box: BoundingBox, magnitude: float, @@ -150,6 +149,7 @@ def estimate_simulation_duration( ) # import here rather than at the module level because openquake is slow to import from empirical.util import openquake_wrapper_vectorized as openquake + ds = np.exp( openquake.oq_run(GMM.AS_16, TectType.ACTIVE_SHALLOW, oq_dataframe, "Ds595")[ "Ds595_mean" @@ -250,11 +250,15 @@ def generate_velocity_model_parameters( rupture_magnitude = total_magnitude(np.array(list(magnitudes.values()))) rrups = { - fault_name: np.interp(magnitudes[fault_name], velocity_model_parameters.rrup_interpolants[:, 0], velocity_model_parameters.rrup_interpolants[:, 1]) + fault_name: np.interp( + magnitudes[fault_name], + velocity_model_parameters.rrup_interpolants[:, 0], + velocity_model_parameters.rrup_interpolants[:, 1], + ) for fault_name, fault in source_config.source_geometries.items() } logger = log_utils.get_logger(__name__) - logger.debug(log_utils.structured_log("computed rrups", rrups=rrups)) + logger.debug("computed rrups", rrups=rrups) initial_fault = source_config.source_geometries[rupture_propagation.initial_fault] max_depth = get_max_depth( diff --git a/workflow/scripts/hf_sim.py b/workflow/scripts/hf_sim.py index 5d47934..9cd6af9 100644 --- a/workflow/scripts/hf_sim.py +++ b/workflow/scripts/hf_sim.py @@ -149,11 +149,7 @@ def hf_simulate_station( hf_sim_input_str = "\n".join(str(line) for line in hf_sim_input) print("---\n" + hf_sim_input_str + "\n---") - logger.info( - log_utils.structured_log( - "running hf", station=name, input=hf_sim_input_str - ) - ) + logger.info("running hf", station=name, input=hf_sim_input_str) output = subprocess.run( str(hf_sim_path), input=hf_sim_input_str, @@ -162,20 +158,14 @@ def hf_simulate_station( stderr=subprocess.PIPE, ) except subprocess.CalledProcessError as e: - logger.error( - log_utils.structured_log( - "hf failed", station=name, stdout=e.stdout, stderr=e.stderr - ) - ) + logger.error("hf failed", station=name, stdout=e.stdout, stderr=e.stderr) raise epicentre_distance = np.fromstring(output.stderr, dtype="f4", sep="\n") logger.info( - log_utils.structured_log( - "hf succeeded", - station=name, - epicentre_distance=epicentre_distance, - stderr=output.stderr, - ) + "hf succeeded", + station=name, + epicentre_distance=epicentre_distance, + stderr=output.stderr, ) if epicentre_distance.size != 1: @@ -194,20 +184,14 @@ def run_hf( Path, typer.Argument(exists=True), ], - station_file: Annotated[ - Path, typer.Argument(exists=True) - ], - out_file: Annotated[ - Path, typer.Argument(file_okay=False) - ], + station_file: Annotated[Path, typer.Argument(exists=True)], + out_file: Annotated[Path, typer.Argument(file_okay=False)], hf_sim_path: Annotated[Path, typer.Option()] = Path( "/EMOD3D/tools/hb_high_binmod_v6.0.3" ), work_directory: Annotated[ Path, - typer.Option( - exists=True, writable=True, file_okay=False - ), + typer.Option(exists=True, writable=True, file_okay=False), ] = Path("/out"), ): """Run the HF (High-Frequency) simulation and generate the HF output file. diff --git a/workflow/scripts/realisation_to_srf.py b/workflow/scripts/realisation_to_srf.py index 5bf3dc6..8446fe3 100644 --- a/workflow/scripts/realisation_to_srf.py +++ b/workflow/scripts/realisation_to_srf.py @@ -229,28 +229,20 @@ def generate_fault_srf( srf_file_path = output_directory / "srf" / (name + ".srf") with open(srf_file_path, "w", encoding="utf-8") as srf_file_handle: logger = log_utils.get_logger(__name__) - logger.info( - log_utils.structured_log("executing command", cmd=" ".join(genslip_cmd)) - ) + logger.info("executing command", cmd=" ".join(genslip_cmd)) try: proc = subprocess.run( genslip_cmd, stdout=srf_file_handle, stderr=subprocess.PIPE, check=True ) except subprocess.CalledProcessError as e: logger.error( - log_utils.structured_log( - "failed", - exception=e.output.decode("utf-8"), - code=e.returncode, - stderr=e.stderr.decode("utf-8"), - ) + "failed", + exception=e.output.decode("utf-8"), + code=e.returncode, + stderr=e.stderr.decode("utf-8"), ) raise - logger.info( - log_utils.structured_log( - "command completed", stderr=proc.stderr.decode("utf-8") - ) - ) + logger.info("command completed", stderr=proc.stderr.decode("utf-8")) def concatenate_csr_arrays(csr_arrays: list[csr_array]) -> csr_array: @@ -427,12 +419,10 @@ def process_fault(fault_name: str) -> None: logger = log_utils.get_logger(__name__) fault_srf.points["tinit"] += time_delay logger.info( - log_utils.structured_log( - "computed delay", - fault_name=fault_name, - delay=time_delay, - srf_min=fault_srf.points["tinit"].min(), - ) + "computed delay", + fault_name=fault_name, + delay=time_delay, + srf_min=fault_srf.points["tinit"].min(), ) srf_file_map[fault_name] = fault_srf @@ -550,10 +540,16 @@ def generate_fault_srfs_parallel( @cli.from_docstring(app) @log_call() def generate_srf( - realisation_ffp: Annotated[Path, typer.Argument(exists=True, readable=True, dir_okay=False)], + realisation_ffp: Annotated[ + Path, typer.Argument(exists=True, readable=True, dir_okay=False) + ], output_srf_filepath: Annotated[Path, typer.Argument(writable=True, dir_okay=False)], - work_directory: Annotated[Path, typer.Option(exists=True, file_okay=False)] = Path("/out"), - genslip_path: Annotated[Path, typer.Option(readable=True, dir_okay=False)] = Path("/EMOD3D/tools/genslip_v5.4.2"), + work_directory: Annotated[Path, typer.Option(exists=True, file_okay=False)] = Path( + "/out" + ), + genslip_path: Annotated[Path, typer.Option(readable=True, dir_okay=False)] = Path( + "/EMOD3D/tools/genslip_v5.4.2" + ), ): """Generate an SRF file from a given realisation specification.