Keep exceptions logs

doc/release_notes.rst

@@ -12,7 +12,12 @@ Upcoming release
 Please add descriptive release notes like in `PyPSA-Eur <https://github.com/PyPSA/pypsa-eur/blob/master/doc/release_notes.rst>`__.
 E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_tests` and in one sentence what it does.
 
-**New Features and major Changes**
+* Keep all traceback in logs. `PR #898 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/898>`__
+
+PyPSA-Earth 0.2.3
+=================
+
+**New Features and major Changes (19th October 2023)**
 
 * Add params: section in rule definition to keep track of changed settings in config.yaml. `PR #823 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/823>`__ and `PR #880 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/880>`__
 

scripts/_helpers.py

@@ -8,6 +8,7 @@
 import logging
 import os
 import subprocess
+import sys
 from pathlib import Path
 
 import country_converter as coco
@@ -22,6 +23,48 @@
 
 REGION_COLS = ["geometry", "name", "x", "y", "country"]
 
+logger = logging.getLogger(__name__)
+
+
+def handle_exception(exc_type, exc_value, exc_traceback):
+    """
+    Customise errors traceback.
+    """
+
+    tb = exc_traceback
+    while tb.tb_next:
+        tb = tb.tb_next
+    flname = tb.tb_frame.f_globals.get("__file__")
+    funcname = tb.tb_frame.f_code.co_name
+
+    if issubclass(exc_type, KeyboardInterrupt):
+        logger.error(
+            "Manual interruption %r, function %r: %s",
+            flname,
+            funcname,
+            exc_value,
+        )
+    else:
+        logger.error(
+            "An error happened in module %r, function %r: %s",
+            flname,
+            funcname,
+            exc_value,
+            exc_info=(exc_type, exc_value, exc_traceback),
+        )
+
+
+def create_logger(logger_name, level=logging.INFO):
+    """
+    Create a logger for a module and adds a handler needed to capture in logs
+    traceback from exceptions emerging during the workflow.
+    """
+
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(level)
+    handler = logging.StreamHandler(stream=sys.stdout)
+    logger.addHandler(handler)
+
 
 def read_osm_config(*args):
     """

scripts/add_electricity.py

@@ -87,19 +87,27 @@
 
 import logging
 import os
+import sys
 
 import geopandas as gpd
 import numpy as np
 import pandas as pd
 import powerplantmatching as pm
 import pypsa
 import xarray as xr
-from _helpers import configure_logging, read_csv_nafix, update_p_nom_max
+from _helpers import (
+    configure_logging,
+    create_logger,
+    handle_exception,
+    read_csv_nafix,
+    update_p_nom_max,
+)
 from powerplantmatching.export import map_country_bus
 
 idx = pd.IndexSlice
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
+sys.excepthook = handle_exception
 
 
 def normed(s):

scripts/add_extra_components.py

@@ -54,11 +54,12 @@
 """
 import logging
 import os
+import sys
 
 import numpy as np
 import pandas as pd
 import pypsa
-from _helpers import configure_logging
+from _helpers import configure_logging, create_logger, handle_exception
 from add_electricity import (
     _add_missing_carriers_from_costs,
     add_nice_carrier_names,
@@ -67,7 +68,8 @@
 
 idx = pd.IndexSlice
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
+sys.excepthook = handle_exception
 
 
 def attach_storageunits(n, costs, config):

scripts/augmented_line_connections.py

@@ -30,19 +30,21 @@
 """
 import logging
 import os
+import sys
 
 import networkx as nx
 import numpy as np
 import pandas as pd
 import pypsa
-from _helpers import configure_logging
+from _helpers import configure_logging, create_logger, handle_exception
 from add_electricity import load_costs
 from base_network import _set_dc_underwater_fraction
 from networkx.algorithms import complement
 from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation
 from pypsa.geo import haversine_pts
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
+sys.excepthook = handle_exception
 
 
 # Functions

scripts/base_network.py

@@ -57,6 +57,7 @@
 """
 import logging
 import os
+import sys
 
 import geopandas as gpd
 import networkx as nx
@@ -67,12 +68,13 @@
 import shapely.prepared
 import shapely.wkt
 import yaml
-from _helpers import configure_logging, read_csv_nafix
+from _helpers import configure_logging, create_logger, handle_exception, read_csv_nafix
 from scipy.sparse import csgraph
 from shapely.geometry import LineString, Point
 from shapely.ops import unary_union
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
+sys.excepthook = handle_exception
 
 
 def _get_oid(df):

scripts/build_bus_regions.py

@@ -44,16 +44,18 @@
 """
 import logging
 import os
+import sys
 
 import geopandas as gpd
 import pandas as pd
 import pypsa
-from _helpers import REGION_COLS, configure_logging
+from _helpers import REGION_COLS, configure_logging, create_logger, handle_exception
 from shapely.geometry import Point
 
 # from scripts.build_shapes import gadm
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
+sys.excepthook = handle_exception
 
 
 def custom_voronoi_partition_pts(points, outline, add_bounds_shape=True, multiplier=5):

scripts/build_cutout.py

@@ -95,13 +95,15 @@
 """
 import logging
 import os
+import sys
 
 import atlite
 import geopandas as gpd
 import pandas as pd
-from _helpers import configure_logging
+from _helpers import configure_logging, create_logger, handle_exception
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
+sys.excepthook = handle_exception
 
 if __name__ == "__main__":
     if "snakemake" not in globals():

scripts/build_demand_profiles.py

@@ -42,6 +42,7 @@
 """
 import logging
 import os
+import sys
 from itertools import product
 
 import geopandas as gpd
@@ -51,11 +52,18 @@
 import pypsa
 import scipy.sparse as sparse
 import xarray as xr
-from _helpers import configure_logging, getContinent, update_p_nom_max
+from _helpers import (
+    configure_logging,
+    create_logger,
+    getContinent,
+    handle_exception,
+    update_p_nom_max,
+)
 from shapely.prepared import prep
 from shapely.validation import make_valid
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
+sys.excepthook = handle_exception
 
 
 def normed(s):

scripts/build_natura_raster.py

@@ -53,16 +53,18 @@
 """
 import logging
 import os
+import sys
 
 import atlite
 import geopandas as gpd
 import numpy as np
 import rasterio as rio
-from _helpers import configure_logging
+from _helpers import configure_logging, create_logger, handle_exception
 from rasterio.features import geometry_mask
 from rasterio.warp import transform_bounds
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
+sys.excepthook = handle_exception
 
 CUTOUT_CRS = "EPSG:4326"
 

scripts/build_osm_network.py

@@ -7,12 +7,15 @@
 
 import logging
 import os
+import sys
 
 import geopandas as gpd
 import numpy as np
 import pandas as pd
 from _helpers import (
     configure_logging,
+    create_logger,
+    handle_exception,
     read_geojson,
     read_osm_config,
     sets_path_to_root,
@@ -22,7 +25,8 @@
 from shapely.ops import linemerge, split
 from tqdm import tqdm
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
+sys.excepthook = handle_exception
 
 
 def line_endings_to_bus_conversion(lines):

scripts/build_powerplants.py

@@ -94,6 +94,7 @@
 """
 import logging
 import os
+import sys
 
 import geopandas as gpd
 import numpy as np
@@ -103,6 +104,8 @@
 import yaml
 from _helpers import (
     configure_logging,
+    create_logger,
+    handle_exception,
     read_csv_nafix,
     to_csv_nafix,
     two_digits_2_name_country,
@@ -112,7 +115,8 @@
 from shapely import wkt
 from shapely.geometry import Point
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
+sys.excepthook = handle_exception
 
 
 def convert_osm_to_pm(filepath_ppl_osm, filepath_ppl_pm):

scripts/build_renewable_profiles.py

@@ -193,6 +193,7 @@
 import functools
 import logging
 import os
+import sys
 import time
 from math import isnan
 
@@ -203,15 +204,22 @@
 import pandas as pd
 import progressbar as pgb
 import xarray as xr
-from _helpers import configure_logging, read_csv_nafix, sets_path_to_root
+from _helpers import (
+    configure_logging,
+    create_logger,
+    handle_exception,
+    read_csv_nafix,
+    sets_path_to_root,
+)
 from add_electricity import load_powerplants
 from dask.distributed import Client, LocalCluster
 from pypsa.geo import haversine
 from shapely.geometry import LineString, Point, box
 
 cc = coco.CountryConverter()
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
+sys.excepthook = handle_exception
 
 COPERNICUS_CRS = "EPSG:4326"
 GEBCO_CRS = "EPSG:4326"

scripts/build_shapes.py

@@ -9,6 +9,7 @@
 import multiprocessing as mp
 import os
 import shutil
+import sys
 import zipfile
 from itertools import takewhile
 from math import ceil
@@ -24,6 +25,8 @@
 import xarray as xr
 from _helpers import (
     configure_logging,
+    create_logger,
+    handle_exception,
     sets_path_to_root,
     three_2_two_digits_country,
     two_2_three_digits_country,
@@ -36,11 +39,11 @@
 from shapely.validation import make_valid
 from tqdm import tqdm
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-
 sets_path_to_root("pypsa-earth")
 
+create_logger(__name__)
+sys.excepthook = handle_exception
+
 
 def get_GADM_filename(country_code):
     """