diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 857010fcd..41c989826 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -12,7 +12,12 @@ Upcoming release Please add descriptive release notes like in `PyPSA-Eur `__. E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_tests` and in one sentence what it does. -**New Features and major Changes** +* Keep all traceback in logs. `PR #898 `__ + +PyPSA-Earth 0.2.3 +================= + +**New Features and major Changes (19th October 2023)** * Add params: section in rule definition to keep track of changed settings in config.yaml. `PR #823 `__ and `PR #880 `__ diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 6d4e5d80d..f5ae8aed4 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -8,6 +8,7 @@ import logging import os import subprocess +import sys from pathlib import Path import country_converter as coco @@ -22,6 +23,47 @@ REGION_COLS = ["geometry", "name", "x", "y", "country"] +logger = logging.getLogger(__name__) + + +def handle_exception(exc_type, exc_value, exc_traceback): + """ + Customise errors traceback. + """ + tb = exc_traceback + while tb.tb_next: + tb = tb.tb_next + flname = tb.tb_frame.f_globals.get("__file__") + funcname = tb.tb_frame.f_code.co_name + + if issubclass(exc_type, KeyboardInterrupt): + logger.error( + "Manual interruption %r, function %r: %s", + flname, + funcname, + exc_value, + ) + else: + logger.error( + "An error happened in module %r, function %r: %s", + flname, + funcname, + exc_value, + exc_info=(exc_type, exc_value, exc_traceback), + ) + + +def create_logger(logger_name, level=logging.INFO): + """ + Create a logger for a module and adds a handler needed to capture in logs + traceback from exceptions emerging during the workflow. + """ + logger = logging.getLogger(logger_name) + logger.setLevel(level) + handler = logging.StreamHandler(stream=sys.stdout) + logger.addHandler(handler) + sys.excepthook = handle_exception + def read_osm_config(*args): """ diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 6cc7b634e..23bf15ad7 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -94,12 +94,12 @@ import powerplantmatching as pm import pypsa import xarray as xr -from _helpers import configure_logging, read_csv_nafix, update_p_nom_max +from _helpers import configure_logging, create_logger, read_csv_nafix, update_p_nom_max from powerplantmatching.export import map_country_bus idx = pd.IndexSlice -logger = logging.getLogger(__name__) +create_logger(__name__) def normed(s): diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 65f21f075..1b2f3a92a 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -58,7 +58,7 @@ import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, create_logger from add_electricity import ( _add_missing_carriers_from_costs, add_nice_carrier_names, @@ -67,7 +67,7 @@ idx = pd.IndexSlice -logger = logging.getLogger(__name__) +create_logger(__name__) def attach_storageunits(n, costs, config): diff --git a/scripts/augmented_line_connections.py b/scripts/augmented_line_connections.py index 4f5dcfb30..7329049db 100644 --- a/scripts/augmented_line_connections.py +++ b/scripts/augmented_line_connections.py @@ -35,14 +35,14 @@ import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, create_logger from add_electricity import load_costs from base_network import _set_dc_underwater_fraction from networkx.algorithms import complement from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation from pypsa.geo import haversine_pts -logger = logging.getLogger(__name__) +create_logger(__name__) # Functions diff --git a/scripts/base_network.py b/scripts/base_network.py index c321f2c29..c302d5b4c 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -67,12 +67,12 @@ import shapely.prepared import shapely.wkt import yaml -from _helpers import configure_logging, read_csv_nafix +from _helpers import configure_logging, create_logger, read_csv_nafix from scipy.sparse import csgraph from shapely.geometry import LineString, Point from shapely.ops import unary_union -logger = logging.getLogger(__name__) +create_logger(__name__) def _get_oid(df): diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 340bbdeac..66b12bbd5 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -48,12 +48,12 @@ import geopandas as gpd import pandas as pd import pypsa -from _helpers import REGION_COLS, configure_logging +from _helpers import REGION_COLS, configure_logging, create_logger from shapely.geometry import Point # from scripts.build_shapes import gadm -logger = logging.getLogger(__name__) +create_logger(__name__) def custom_voronoi_partition_pts(points, outline, add_bounds_shape=True, multiplier=5): diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index ea72aa294..05aa842f0 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -99,9 +99,10 @@ import atlite import geopandas as gpd import pandas as pd -from _helpers import configure_logging +from _helpers import configure_logging, create_logger + +create_logger(__name__) -logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index ae031da47..4e57dc1e8 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -51,11 +51,11 @@ import pypsa import scipy.sparse as sparse import xarray as xr -from _helpers import configure_logging, getContinent, update_p_nom_max +from _helpers import configure_logging, create_logger, getContinent, update_p_nom_max from shapely.prepared import prep from shapely.validation import make_valid -logger = logging.getLogger(__name__) +create_logger(__name__) def normed(s): diff --git a/scripts/build_natura_raster.py b/scripts/build_natura_raster.py index febedc98f..98406b6ea 100644 --- a/scripts/build_natura_raster.py +++ b/scripts/build_natura_raster.py @@ -58,11 +58,12 @@ import geopandas as gpd import numpy as np import rasterio as rio -from _helpers import configure_logging +from _helpers import configure_logging, create_logger from rasterio.features import geometry_mask from rasterio.warp import transform_bounds -logger = logging.getLogger(__name__) +create_logger(__name__) + CUTOUT_CRS = "EPSG:4326" diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index dd70cbc1b..fa71e5100 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -13,6 +13,7 @@ import pandas as pd from _helpers import ( configure_logging, + create_logger, read_geojson, read_osm_config, sets_path_to_root, @@ -22,7 +23,7 @@ from shapely.ops import linemerge, split from tqdm import tqdm -logger = logging.getLogger(__name__) +create_logger(__name__) def line_endings_to_bus_conversion(lines): diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index b2f14d582..bfced3ce6 100644 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -103,6 +103,7 @@ import yaml from _helpers import ( configure_logging, + create_logger, read_csv_nafix, to_csv_nafix, two_digits_2_name_country, @@ -112,7 +113,7 @@ from shapely import wkt from shapely.geometry import Point -logger = logging.getLogger(__name__) +create_logger(__name__) def convert_osm_to_pm(filepath_ppl_osm, filepath_ppl_pm): diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 99f3a79ba..ce1128f16 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -203,7 +203,7 @@ import pandas as pd import progressbar as pgb import xarray as xr -from _helpers import configure_logging, read_csv_nafix, sets_path_to_root +from _helpers import configure_logging, create_logger, read_csv_nafix, sets_path_to_root from add_electricity import load_powerplants from dask.distributed import Client, LocalCluster from pypsa.geo import haversine @@ -211,7 +211,8 @@ cc = coco.CountryConverter() -logger = logging.getLogger(__name__) +create_logger(__name__) + COPERNICUS_CRS = "EPSG:4326" GEBCO_CRS = "EPSG:4326" diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index dd8d8edcb..659fa2da4 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -24,6 +24,7 @@ import xarray as xr from _helpers import ( configure_logging, + create_logger, sets_path_to_root, three_2_two_digits_country, two_2_three_digits_country, @@ -36,11 +37,10 @@ from shapely.validation import make_valid from tqdm import tqdm -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - sets_path_to_root("pypsa-earth") +create_logger(__name__) + def get_GADM_filename(country_code): """ diff --git a/scripts/clean_osm_data.py b/scripts/clean_osm_data.py index a3d1d22f9..f7544bc83 100644 --- a/scripts/clean_osm_data.py +++ b/scripts/clean_osm_data.py @@ -12,9 +12,15 @@ import numpy as np import pandas as pd import reverse_geocode as rg -from _helpers import REGION_COLS, configure_logging, save_to_geojson, to_csv_nafix - -logger = logging.getLogger(__name__) +from _helpers import ( + REGION_COLS, + configure_logging, + create_logger, + save_to_geojson, + to_csv_nafix, +) + +create_logger(__name__) def prepare_substation_df(df_all_substations): diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 8a3674895..3f783e7cc 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -135,6 +135,7 @@ from _helpers import ( REGION_COLS, configure_logging, + create_logger, get_aggregation_strategies, sets_path_to_root, update_p_nom_max, @@ -151,7 +152,7 @@ idx = pd.IndexSlice -logger = logging.getLogger(__name__) +create_logger(__name__) def normed(x): diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py index 24afa4587..9498f215a 100644 --- a/scripts/download_osm_data.py +++ b/scripts/download_osm_data.py @@ -31,10 +31,10 @@ import shutil from pathlib import Path -from _helpers import configure_logging, read_osm_config +from _helpers import configure_logging, create_logger, read_osm_config from earth_osm import eo -logger = logging.getLogger(__name__) +create_logger(__name__) def country_list_to_geofk(country_list): diff --git a/scripts/make_statistics.py b/scripts/make_statistics.py index d87eb71c2..b883f478b 100644 --- a/scripts/make_statistics.py +++ b/scripts/make_statistics.py @@ -32,11 +32,13 @@ import pandas as pd import pypsa import xarray as xr -from _helpers import mock_snakemake, sets_path_to_root, to_csv_nafix +from _helpers import create_logger, mock_snakemake, sets_path_to_root, to_csv_nafix from build_test_configs import create_test_config from ruamel.yaml import YAML from shapely.validation import make_valid +create_logger(__name__) + def _multi_index_scen(rulename, keys): return pd.MultiIndex.from_product([[rulename], keys], names=["rule", "key"]) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 4ecf727bc..f1e2f1399 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -47,11 +47,12 @@ import pandas as pd import pypsa from _helpers import configure_logging -from add_electricity import load_costs, update_transmission_costs +from add_electricity import create_logger, load_costs, update_transmission_costs idx = pd.IndexSlice -logger = logging.getLogger(__name__) +create_logger(__name__) + opt_name = {"Store": "e", "Line": "s", "Transformer": "s"} diff --git a/scripts/monte_carlo.py b/scripts/monte_carlo.py index f60180236..3a526063d 100644 --- a/scripts/monte_carlo.py +++ b/scripts/monte_carlo.py @@ -70,12 +70,12 @@ import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, create_logger from pyDOE2 import lhs from scipy.stats import qmc from solve_network import * -logger = logging.getLogger(__name__) +create_logger(__name__) def monte_carlo_sampling_pydoe2( diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 4084e0881..f72d721aa 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -29,6 +29,7 @@ aggregate_costs, aggregate_p, configure_logging, + create_logger, load_network_for_plots, ) from matplotlib.legend_handler import HandlerPatch @@ -36,7 +37,7 @@ to_rgba = mpl.colors.colorConverter.to_rgba -logger = logging.getLogger(__name__) +create_logger(__name__) def make_handler_map_to_scale_circles_as_in(ax, dont_resize_actively=False): diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 6da5dd4f9..13215e674 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -21,9 +21,9 @@ import matplotlib.pyplot as plt import pandas as pd -from _helpers import configure_logging +from _helpers import configure_logging, create_logger -logger = logging.getLogger(__name__) +create_logger(__name__) def rename_techs(label): diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 90da72585..2afa7a083 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -66,12 +66,12 @@ import pandas as pd import pypsa import requests -from _helpers import configure_logging +from _helpers import configure_logging, create_logger from add_electricity import load_costs, update_transmission_costs idx = pd.IndexSlice -logger = logging.getLogger(__name__) +create_logger(__name__) def download_emission_data(): diff --git a/scripts/retrieve_databundle_light.py b/scripts/retrieve_databundle_light.py index 59a66346d..e18e7e62b 100644 --- a/scripts/retrieve_databundle_light.py +++ b/scripts/retrieve_databundle_light.py @@ -89,12 +89,13 @@ from _helpers import ( configure_logging, create_country_list, + create_logger, progress_retrieve, sets_path_to_root, ) from google_drive_downloader import GoogleDriveDownloader as gdd -logger = logging.getLogger(__name__) +create_logger(__name__) def load_databundle_config(config): diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 4e2d1c2c0..7a2f01619 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -93,7 +93,12 @@ import pandas as pd import pypsa import scipy as sp -from _helpers import configure_logging, get_aggregation_strategies, update_p_nom_max +from _helpers import ( + configure_logging, + create_logger, + get_aggregation_strategies, + update_p_nom_max, +) from add_electricity import load_costs from cluster_network import cluster_regions, clustering_for_n_clusters from pypsa.clustering.spatial import ( @@ -107,7 +112,7 @@ sys.settrace -logger = logging.getLogger(__name__) +create_logger(__name__) def simplify_network_to_base_voltage(n, linetype, base_voltage): diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 0a00f89d7..589656e2d 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -85,7 +85,7 @@ import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, create_logger from pypsa.descriptors import get_switchable_as_dense as get_as_dense from pypsa.linopf import ( define_constraints, @@ -97,7 +97,7 @@ network_lopf, ) -logger = logging.getLogger(__name__) +create_logger(__name__) def prepare_network(n, solve_opts):