Merge pull request #304 from PyPSA/ci-review-cherries

cherry-pick from #282

.github/workflows/ci.yaml

@@ -102,7 +102,5 @@ jobs:
         run: |
           conda activate pypsa-eur
           conda list
-          cp test/config.overnight.yaml config.yaml
-          snakemake -call
-          cp test/config.myopic.yaml config.yaml
-          snakemake -call
+          snakemake -call --configfile test/config.overnight.yaml
+          snakemake -call --configfile test/config.myopic.yaml

.gitignore

@@ -27,6 +27,10 @@ gurobi.log
 /data/retro/tabula-calculator-calcsetbuilding.csv
 /data/nuts*
 data/gas_network/scigrid-gas/
+data/costs_*.csv
+
+dask-worker-space
+
 dask-worker-space/
 publications.jrc.ec.europa.eu/
 

Snakefile

@@ -259,9 +259,9 @@ rule build_biomass_potentials:
         enspreso_biomass=HTTP.remote("https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx", keep_local=True),
         nuts2="data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21
         regions_onshore=pypsaeur("resources/regions_onshore_elec_s{simpl}_{clusters}.geojson"),
-        nuts3_population="../pypsa-eur/data/bundle/nama_10r_3popgdp.tsv.gz",
-        swiss_cantons="../pypsa-eur/data/bundle/ch_cantons.csv",
-        swiss_population="../pypsa-eur/data/bundle/je-e-21.03.02.xls",
+        nuts3_population=pypsaeur("data/bundle/nama_10r_3popgdp.tsv.gz"),
+        swiss_cantons=pypsaeur("data/bundle/ch_cantons.csv"),
+        swiss_population=pypsaeur("data/bundle/je-e-21.03.02.xls"),
         country_shapes=pypsaeur('resources/country_shapes.geojson')
     output:
         biomass_potentials_all='resources/biomass_potentials_all_s{simpl}_{clusters}.csv',

scripts/add_brownfield.py

@@ -12,7 +12,6 @@
 
 from add_existing_baseyear import add_build_year_to_new_assets
 from helper import override_component_attrs, update_config_with_sector_opts
-from solve_network import basename
 
 
 def add_brownfield(n, n_p, year):
@@ -99,7 +98,7 @@ def add_brownfield(n, n_p, year):
             pipe_capacity = n.links.loc[gas_pipes_i, 'p_nom']
             # already retrofitted capacity from gas -> H2
             already_retrofitted = (n.links.loc[h2_retrofitted_fixed_i, 'p_nom']
-                                   .rename(lambda x: basename(x).replace(fr, to)).groupby(level=0).sum())
+                                   .rename(lambda x: x.split("-2")[0].replace(fr, to)).groupby(level=0).sum())
             remaining_capacity = pipe_capacity - CH4_per_H2 * already_retrofitted.reindex(index=pipe_capacity.index).fillna(0)
             n.links.loc[gas_pipes_i, "p_nom"] = remaining_capacity
         else:

scripts/plot_network.py

@@ -62,6 +62,10 @@ def plot_map(network, components=["links", "stores", "storage_units", "generator
 
     for comp in components:
         df_c = getattr(n, comp)
+
+        if df_c.empty:
+            continue
+
         df_c["nice_group"] = df_c.carrier.map(rename_techs_tyndp)
 
         attr = "e_nom_opt" if comp == "stores" else "p_nom_opt"
@@ -240,7 +244,7 @@ def group_pipes(df, drop_direction=False):
         axis=1
     )
     # group pipe lines connecting the same buses and rename them for plotting
-    pipe_capacity = df["p_nom_opt"].groupby(level=0).sum()
+    pipe_capacity = df.groupby(level=0).agg({"p_nom_opt": sum, "bus0": "first", "bus1": "first"})
 
     return pipe_capacity
 
@@ -276,13 +280,16 @@ def plot_h2_map(network, regions):
     # drop all links which are not H2 pipelines
     n.links.drop(n.links.index[~n.links.carrier.str.contains("H2 pipeline")], inplace=True)
 
-    h2_new = n.links.loc[n.links.carrier=="H2 pipeline"]
-    h2_retro = n.links.loc[n.links.carrier=='H2 pipeline retrofitted']
+    h2_new = n.links[n.links.carrier=="H2 pipeline"]
+    h2_retro = n.links[n.links.carrier=='H2 pipeline retrofitted']
 
     if snakemake.config['foresight'] == 'myopic':
         # sum capacitiy for pipelines from different investment periods
         h2_new = group_pipes(h2_new)
-        h2_retro = group_pipes(h2_retro, drop_direction=True).reindex(h2_new.index).fillna(0)
+
+        if not h2_retro.empty:
+            h2_retro = group_pipes(h2_retro, drop_direction=True).reindex(h2_new.index).fillna(0)
+
 
     if not h2_retro.empty:
 

scripts/prepare_sector_network.py

@@ -458,7 +458,9 @@ def remove_elec_base_techs(n):
     for c in n.iterate_components(snakemake.config["pypsa_eur"]):
         to_keep = snakemake.config["pypsa_eur"][c.name]
         to_remove = pd.Index(c.df.carrier.unique()).symmetric_difference(to_keep)
-        print("Removing", c.list_name, "with carrier", to_remove)
+        if to_remove.empty:
+            continue
+        logger.info(f"Removing {c.list_name} with carrier {list(to_remove)}")
         names = c.df.index[c.df.carrier.isin(to_remove)]
         n.mremove(c.name, names)
         n.carriers.drop(to_remove, inplace=True, errors="ignore")
@@ -469,8 +471,10 @@ def remove_non_electric_buses(n):
     """
     remove buses from pypsa-eur with carriers which are not AC buses
     """
-    print("drop buses from PyPSA-Eur with carrier: ", n.buses[~n.buses.carrier.isin(["AC", "DC"])].carrier.unique())
-    n.buses = n.buses[n.buses.carrier.isin(["AC", "DC"])]
+    to_drop = list(n.buses.query("carrier not in ['AC', 'DC']").carrier.unique())
+    if to_drop:
+        logger.info(f"Drop buses from PyPSA-Eur with carrier: {to_drop}")
+        n.buses = n.buses[n.buses.carrier.isin(["AC", "DC"])]
 
 
 def patch_electricity_network(n):
@@ -533,6 +537,8 @@ def add_co2_tracking(n, options):
         bus=spatial.co2.nodes
     )
 
+    n.add("Carrier", "co2 stored")
+
     if options['co2_vent']:
 
         n.madd("Link",
@@ -821,12 +827,12 @@ def insert_electricity_distribution_grid(n, costs):
     # TODO pop_layout?
     # TODO options?
 
-    print("Inserting electricity distribution grid with investment cost factor of",
-          options['electricity_distribution_grid_cost_factor'])
+    cost_factor = options['electricity_distribution_grid_cost_factor']
+
+    logger.info(f"Inserting electricity distribution grid with investment cost factor of {cost_factor:.2f}")
 
     nodes = pop_layout.index
 
-    cost_factor = options['electricity_distribution_grid_cost_factor']
 
     n.madd("Bus",
         nodes + " low voltage",
@@ -941,7 +947,7 @@ def insert_gas_distribution_costs(n, costs):
 
     f_costs = options['gas_distribution_grid_cost_factor']
 
-    print("Inserting gas distribution grid with investment cost factor of", f_costs)
+    logger.info(f"Inserting gas distribution grid with investment cost factor of {f_costs}")
 
     capital_cost = costs.loc['electricity distribution grid']["fixed"] * f_costs
 
@@ -1317,7 +1323,7 @@ def add_land_transport(n, costs):
 
     total_share = fuel_cell_share + electric_share + ice_share
     if total_share != 1:
-        logger.warning(f"Total land transport shares sum up to {total_share*100}%, corresponding to increased or decreased demand assumptions.")
+        logger.warning(f"Total land transport shares sum up to {total_share:.2%}, corresponding to increased or decreased demand assumptions.")
 
     logger.info(f"FCEV share: {fuel_cell_share*100}%")
     logger.info(f"EV share: {electric_share*100}%")
@@ -1487,7 +1493,7 @@ def add_heat(n, costs):
     # exogenously reduce space heat demand
     if options["reduce_space_heat_exogenously"]:
         dE = get(options["reduce_space_heat_exogenously_factor"], investment_year)
-        print(f"assumed space heat reduction of {dE*100} %")
+        logger.info(f"assumed space heat reduction of {dE:.2%}")
         for sector in sectors:
             heat_demand[sector + " space"] = (1 - dE) * heat_demand[sector + " space"]
 
@@ -1847,10 +1853,9 @@ def create_nodes_for_heat_sector():
     diff = (urban_fraction * central_fraction) - dist_fraction_node
     progress = get(options["district_heating"]["progress"], investment_year)
     dist_fraction_node += diff * progress
-    print(
-        "The current district heating share compared to the maximum",
-        f"possible is increased by a progress factor of\n{progress}",
-        f"resulting in a district heating share of\n{dist_fraction_node}"
+    logger.info(
+        f"Increase district heating share by a progress factor of {progress:.2%} "
+        f"resulting in new average share of {dist_fraction_node.mean():.2%}"
     )
 
     return nodes, dist_fraction_node, urban_fraction
@@ -2188,7 +2193,7 @@ def add_industry(n, costs):
 
     total_share = shipping_hydrogen_share + shipping_methanol_share + shipping_oil_share
     if total_share != 1:
-        logger.warning(f"Total shipping shares sum up to {total_share*100}%, corresponding to increased or decreased demand assumptions.")
+        logger.warning(f"Total shipping shares sum up to {total_share:.2%}, corresponding to increased or decreased demand assumptions.")
 
     domestic_navigation = pop_weighted_energy_totals.loc[nodes, "total domestic navigation"].squeeze()
     international_navigation = pd.read_csv(snakemake.input.shipping_demand, index_col=0).squeeze()
@@ -2557,7 +2562,7 @@ def add_agriculture(n, costs):
 
     total_share = electric_share + oil_share
     if total_share != 1:
-        logger.warning(f"Total agriculture machinery shares sum up to {total_share*100}%, corresponding to increased or decreased demand assumptions.")
+        logger.warning(f"Total agriculture machinery shares sum up to {total_share:.2%}, corresponding to increased or decreased demand assumptions.")
 
     machinery_nodal_energy = pop_weighted_energy_totals.loc[nodes, "total agriculture machinery"]
 
@@ -2635,7 +2640,7 @@ def maybe_adjust_costs_and_potentials(n, opts):
                     else:
                         sel = c.df.carrier.str.contains(carrier)
                     c.df.loc[sel,attr] *= factor
-            print("changing", attr , "for", carrier, "by factor", factor)
+            logger.info(f"changing {attr} for {carrier} by factor {factor}")
 
 
 # TODO this should rather be a config no wildcard
@@ -2880,7 +2885,7 @@ def set_temporal_aggregation(n, opts, solver_name):
     for o in opts:
         if o[:4] == "wave":
             wave_cost_factor = float(o[4:].replace("p", ".").replace("m", "-"))
-            print("Including wave generators with cost factor of", wave_cost_factor)
+            logger.info(f"Including wave generators with cost factor of {wave_cost_factor}")
             add_wave(n, wave_cost_factor)
         if o[:4] == "dist":
             options['electricity_distribution_grid'] = True
@@ -2949,7 +2954,7 @@ def set_temporal_aggregation(n, opts, solver_name):
         limit = o[o.find("Co2L")+4:]
         limit = float(limit.replace("p", ".").replace("m", "-"))
         break
-    print("Add CO2 limit from", limit_type)
+    logger.info(f"Add CO2 limit from {limit_type}")
     add_co2limit(n, Nyears, limit)
 
     for o in opts: