Merge pull request #577 from davide-f/create_scenario_management

Create scenario management

config.default.yaml

@@ -10,6 +10,11 @@ logging:
   level: INFO
   format: "%(levelname)s:%(name)s:%(message)s"
 
+run:
+  name: "" # use this to keep track of runs with different settings
+  shared_cutouts: true  # set to true to share the default cutout(s) across runs
+                        # Note: value false requires build_cutout to be enabled
+
 scenario:
   simpl: ['']
   ll: ['copt']

config.tutorial.yaml

@@ -9,6 +9,9 @@ logging:
   level: INFO
   format: "%(levelname)s:%(name)s:%(message)s"
 
+run:
+  name: ""
+
 scenario:
   simpl: ['']
   ll: ['copt']

doc/configuration.rst

@@ -35,13 +35,24 @@ Top-level configuration
 
 .. _scenario:
 
-``scenario``
-============
+``run``
+=======
 
 It is common conduct to analyse energy system optimisation models for **multiple scenarios** for a variety of reasons,
 e.g. assessing their sensitivity towards changing the temporal and/or geographical resolution or investigating how
 investment changes as more ambitious greenhouse-gas emission reduction targets are applied.
 
+The ``run`` section is used for running and storing scenarios with different configurations which are not covered by :ref:`wildcards`. It determines the path at which resources, networks and results are stored. Therefore the user can run different configurations within the same directory. If a run with a non-empty name should use cutouts shared across runs, set ``shared_cutouts`` to `true`.    
+
+.. literalinclude:: ../config.default.yaml
+   :language: yaml
+   :start-at: run:
+   :end-before: scenario:
+
+
+``scenario``
+============
+
 The ``scenario`` section is an extraordinary section of the config file
 that is strongly connected to the :ref:`wildcards` and is designed to
 facilitate running multiple scenarios through a single command

doc/release_notes.rst

@@ -57,6 +57,8 @@ Upcoming Release
 
 * Improve parallel capabilities of build_shapes to enable parallelization even within a country shape `PR #575 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/575>`__
 
+* Add pypsa-eur scenario management `PR #577 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/577>`__
+
 PyPSA-Earth 0.1.0
 =================
 

scripts/add_electricity.py

@@ -308,7 +308,7 @@ def attach_load(
     logger.info(f"Load data scaled with scalling factor {scale}.")
     gegis_load["Electricity demand"] *= scale
     shapes = gpd.read_file(admin_shapes).set_index("GADM_ID")
-    shapes.loc[:, "geometry"] = shapes["geometry"].apply(lambda x: make_valid(x))
+    shapes["geometry"] = shapes["geometry"].apply(lambda x: make_valid(x))
 
     def upsample(cntry, group):
         """

scripts/build_osm_network.py

@@ -84,10 +84,9 @@ def add_line_endings_tosubstations(substations, lines):
     bus_e["bus_id"] = lines["line_id"].astype(str) + "_e"
     bus_e["dc"] = ~is_ac
 
-    bus_all = pd.concat([bus_s, bus_e], ignore_index=True)
+    bus_all = pd.concat([bus_s, bus_e], ignore_index=True).set_crs(substations.crs)
     # Assign index to bus_id
-    bus_all.loc[:, "bus_id"] = bus_all.index
-    buses = bus_all
+    bus_all["bus_id"] = bus_all.index
 
     # Add NaN as default
     bus_all["station_id"] = np.nan

scripts/build_renewable_profiles.py

@@ -582,7 +582,7 @@ def estimate_bus_loss(data_column, tech):
             underwater_fraction = []
             for bus in buses:
                 p = centre_of_mass.sel(bus=bus).data
-                line = LineString([p, regions.loc[bus, ["x", "y"]]])
+                line = LineString([tuple(p), tuple(regions.loc[bus, ["x", "y"]])])
                 frac = line.intersection(offshore_shape).length / line.length
                 underwater_fraction.append(frac)
 

scripts/build_shapes.py

@@ -661,34 +661,6 @@ def add_gdp_data(
             df_gadm.loc[i, "gdp"] = _sum_raster_over_mask(df_gadm.geometry.loc[i], src)
     return df_gadm
 
-    # for index, row in tqdm(df_gadm.iterrows(), index=df_gadm.shape[0]):
-    #     # select the desired area of the raster corresponding to each polygon
-    #     # Approximation: the gdp is measured excluding the pixels
-    #     #   where the border of the shape lays. This may affect the computation
-    #     #   but it is conservative and avoids considering multiple times the same
-    #     #   pixels
-    #     out_image, out_transform = generalized_mask(src,
-    #                                                 row["geometry"],
-    #                                                 all_touched=True,
-    #                                                 invert=False,
-    #                                                 nodata=0.0)
-    #     # out_image_int, out_transform = mask(src,
-    #     #                                row["geometry"],
-    #     #                                all_touched=False,
-    #     #                                invert=False,
-    #     #                                nodata=0.0)
-
-    #     # calculate total gdp in the selected geometry
-    #     gdp_by_geom = np.nansum(out_image)
-    #     # gdp_by_geom = out_image.sum()/2 + out_image_int.sum()/2
-
-    #     if out_logging == True:
-    #         logger.info("Stage 4/4 GDP: shape: " + str(index) +
-    #                      " out of " + str(df_gadm.shape[0]))
-
-    #     # update the gdp data in the dataset
-    #     df_gadm.loc[index, "gdp"] = gdp_by_geom
-
 
 def _init_process_pop(df_gadm_, year_, worldpop_method_):
     global df_gadm, year, worldpop_method
@@ -722,6 +694,7 @@ def _process_func_pop(gadm_idxs):
 
     return df_gadm_subset
 
+
 # Auxiliary function to download WorldPop data in a parallel way
 def _process_func_download_pop(c_code):
     WorldPop_inputfile, WorldPop_filename = download_WorldPop(

scripts/clean_osm_data.py

@@ -188,7 +188,7 @@ def filter_voltage(df, threshold_voltage=35000):
     df = df.dropna(subset=["voltage"])  # Drop any row with Voltage = N/A
 
     # convert voltage to int
-    df.loc[:, "voltage"] = df["voltage"].astype(int)
+    df["voltage"] = df["voltage"].astype(int)
 
     # keep only lines with a voltage no lower than than threshold_voltage
     df = df[df.voltage >= threshold_voltage]
@@ -201,7 +201,7 @@ def finalize_substation_types(df_all_substations):
     Specify bus_id and voltage columns as integer
     """
     df_all_substations["bus_id"] = df_all_substations["bus_id"].astype(int)
-    df_all_substations.loc[:, "voltage"] = df_all_substations["voltage"].astype(int)
+    df_all_substations["voltage"] = df_all_substations["voltage"].astype(int)
 
     return df_all_substations
 

test/config.custom.yaml

@@ -6,6 +6,10 @@
 retrieve_databundle: # required to be "false" for nice CI test output
   show_progress: false
 
+run:
+  name: "custom"
+  shared_cutouts: true # set to true to share the default cutout(s) across runs
+
 download_osm_data_nprocesses: 4
 
 enable: