follow-up to PyPSA#726: import country converter

fix in add_heating_capacities_installed_before_baseyear to account for case when there is no rural heating demand for some nodes in network [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci simplify if-else expression fix in co2_emissions_year function and correspondigly in plot_summary and snakemake rule. Also changes to historical_emissions in plot_summary since some countries are not in eea dataset. [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Update scripts/plot_summary.py Co-authored-by: Fabian Hofmann <hofmann@fias.uni-frankfurt.de> build_carbon_budget: parse input_co2 apply protected() to zenodo downloads; only retrieve monthly fuel/CO2 prices when necessary [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci added basic egs potential script pre-testrun version added to prepare_sector_network added new params to default config, and fixes cleanup and renaming changed default config to pessimism on egs added minor cleanup correcting local adjustment [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci changed config name to egs_performant, added saturation check cleanup of saturation check [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci added source for FOM [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci changed overlap indexing, general cleanup [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci added release notes Fix bug preventing inclusion of custom_powerplants (PyPSA#732) Fix a typo in the execution call [pre-commit.ci] pre-commit autoupdate updates: - [github.com/psf/black: 23.7.0 → 23.9.1](psf/black@23.7.0...23.9.1) Move all rule to main Snakefile Move all rule down to other rules added dialogue to ensure desired purge reverted to original pypsa-eur
LukasFrankenQ · Sep 27, 2023 · c1388b3 · c1388b3
1 parent 59d422a
commit c1388b3
Show file tree

Hide file tree

Showing 17 changed files with 141 additions and 72 deletions.
diff --git a/.gitignore b/.gitignore
@@ -28,18 +28,18 @@ dconf
 /data/links_p_nom.csv
 /data/*totals.csv
 /data/biomass*
-/data/emobility/
-/data/eea*
-/data/jrc*
+/data/bundle-sector/emobility/
+/data/bundle-sector/eea*
+/data/bundle-sector/jrc*
 /data/heating/
-/data/eurostat*
+/data/bundle-sector/eurostat*
 /data/odyssee/
 /data/transport_data.csv
-/data/switzerland*
+/data/bundle-sector/switzerland*
 /data/.nfs*
-/data/Industrial_Database.csv
+/data/bundle-sector/Industrial_Database.csv
 /data/retro/tabula-calculator-calcsetbuilding.csv
-/data/nuts*
+/data/bundle-sector/nuts*
 data/gas_network/scigrid-gas/
 data/costs_*.csv
 

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -51,7 +51,7 @@ repos:
 
   # Formatting with "black" coding style
 - repo: https://github.com/psf/black
-  rev: 23.7.0
+  rev: 23.9.1
   hooks:
       # Format Python files
   - id: black

diff --git a/Snakefile b/Snakefile
@@ -66,13 +66,23 @@ if config["foresight"] == "myopic":
     include: "rules/solve_myopic.smk"
 
 
+rule all:
+    input:
+        RESULTS + "graphs/costs.pdf",
+    default_target: True
+
+
 rule purge:
-    message:
-        "Purging generated resources, results and docs. Downloads are kept."
     run:
-        rmtree("resources/", ignore_errors=True)
-        rmtree("results/", ignore_errors=True)
-        rmtree("doc/_build", ignore_errors=True)
+        import builtins
+        do_purge = builtins.input("Do you really want to delete all generated resources, \nresults and docs (downloads are kept)? [y/N] ")
+        if do_purge == "y":
+            rmtree("resources/", ignore_errors=True)
+            rmtree("results/", ignore_errors=True)
+            rmtree("doc/_build", ignore_errors=True)
+            print("Purging generated resources, results and docs. Downloads are kept.")
+        else:
+            raise Exception(f"Input {do_purge}. Aborting purge.")
 
 
 rule dag:

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
@@ -16,6 +16,15 @@ Upcoming Release
 
 * The minimum capacity for renewable generators when using the myopic option has been fixed.
 
+* Files downloaded from zenodo are now write-protected to prevent accidental re-download.
+
+* Files extracted from sector-coupled data bundle have been moved from ``data/`` to ``data/sector-bundle``.
+
+
+**Bugs and Compatibility**
+
+* A bug preventing custom powerplants specified in ``data/custom_powerplants.csv`` was fixed. (https://github.com/PyPSA/pypsa-eur/pull/732)
+
 PyPSA-Eur 0.8.1 (27th July 2023)
 ================================
 

diff --git a/doc/tutorial_sector.rst b/doc/tutorial_sector.rst
@@ -59,7 +59,7 @@ To run an overnight / greenfiled scenario with the specifications above, run
 
 .. code:: bash
 
-    snakemake -call --configfile config/test/config.overnight.yaml all
+    snakemake -call all --configfile config/test/config.overnight.yaml
 
 which will result in the following *additional* jobs ``snakemake`` wants to run
 on top of those already included in the electricity-only tutorial:
@@ -318,7 +318,7 @@ To run a myopic foresight scenario with the specifications above, run
 
 .. code:: bash
 
-    snakemake -call --configfile config/test/config.myopic.yaml all
+    snakemake -call all --configfile config/test/config.myopic.yaml
 
 which will result in the following *additional* jobs ``snakemake`` wants to run:
 

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
@@ -350,7 +350,9 @@ rule add_electricity:
         hydro_capacities=ancient("data/bundle/hydro_capacities.csv"),
         geth_hydro_capacities="data/geth2015_hydro_capacities.csv",
         unit_commitment="data/unit_commitment.csv",
-        fuel_price=RESOURCES + "monthly_fuel_price.csv",
+        fuel_price=RESOURCES + "monthly_fuel_price.csv"
+        if config["conventional"]["dynamic_fuel_price"]
+        else [],
         load=RESOURCES + "load.csv",
         nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
     output:
@@ -478,7 +480,7 @@ rule prepare_network:
     input:
         RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc",
         tech_costs=COSTS,
-        co2_price=RESOURCES + "co2_price.csv",
+        co2_price=lambda w: RESOURCES + "co2_price.csv" if "Ept" in w.opts else [],
     output:
         RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
     log:

diff --git a/rules/build_sector.smk b/rules/build_sector.smk
@@ -242,9 +242,9 @@ rule build_energy_totals:
         energy=config["energy"],
     input:
         nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
-        co2="data/eea/UNFCCC_v23.csv",
-        swiss="data/switzerland-sfoe/switzerland-new_format.csv",
-        idees="data/jrc-idees-2015",
+        co2="data/bundle-sector/eea/UNFCCC_v23.csv",
+        swiss="data/bundle-sector/switzerland-sfoe/switzerland-new_format.csv",
+        idees="data/bundle-sector/jrc-idees-2015",
         district_heat_share="data/district_heat_share.csv",
         eurostat=input_eurostat,
     output:
@@ -272,7 +272,7 @@ rule build_biomass_potentials:
             "https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx",
             keep_local=True,
         ),
-        nuts2="data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",  # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21
+        nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",  # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
         nuts3_population=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"),
         swiss_cantons=ancient("data/bundle/ch_cantons.csv"),
@@ -366,7 +366,7 @@ if not config["sector"]["regional_co2_sequestration_potential"]["enable"]:
 
 rule build_salt_cavern_potentials:
     input:
-        salt_caverns="data/h2_salt_caverns_GWh_per_sqkm.geojson",
+        salt_caverns="data/bundle-sector/h2_salt_caverns_GWh_per_sqkm.geojson",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
         regions_offshore=RESOURCES + "regions_offshore_elec_s{simpl}_{clusters}.geojson",
     output:
@@ -388,7 +388,7 @@ rule build_ammonia_production:
     params:
         countries=config["countries"],
     input:
-        usgs="data/myb1-2017-nitro.xls",
+        usgs="data/bundle-sector/myb1-2017-nitro.xls",
     output:
         ammonia_production=RESOURCES + "ammonia_production.csv",
     threads: 1
@@ -410,7 +410,7 @@ rule build_industry_sector_ratios:
         ammonia=config["sector"].get("ammonia", False),
     input:
         ammonia_production=RESOURCES + "ammonia_production.csv",
-        idees="data/jrc-idees-2015",
+        idees="data/bundle-sector/jrc-idees-2015",
     output:
         industry_sector_ratios=RESOURCES + "industry_sector_ratios.csv",
     threads: 1
@@ -432,8 +432,8 @@ rule build_industrial_production_per_country:
         countries=config["countries"],
     input:
         ammonia_production=RESOURCES + "ammonia_production.csv",
-        jrc="data/jrc-idees-2015",
-        eurostat="data/eurostat-energy_balances-may_2018_edition",
+        jrc="data/bundle-sector/jrc-idees-2015",
+        eurostat="data/bundle-sector/eurostat-energy_balances-may_2018_edition",
     output:
         industrial_production_per_country=RESOURCES
         + "industrial_production_per_country.csv",
@@ -483,7 +483,7 @@ rule build_industrial_distribution_key:
     input:
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
         clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
-        hotmaps_industrial_database="data/Industrial_Database.csv",
+        hotmaps_industrial_database="data/bundle-sector/Industrial_Database.csv",
     output:
         industrial_distribution_key=RESOURCES
         + "industrial_distribution_key_elec_s{simpl}_{clusters}.csv",
@@ -558,7 +558,7 @@ rule build_industrial_energy_demand_per_country_today:
         countries=config["countries"],
         industry=config["industry"],
     input:
-        jrc="data/jrc-idees-2015",
+        jrc="data/bundle-sector/jrc-idees-2015",
         ammonia_production=RESOURCES + "ammonia_production.csv",
         industrial_production_per_country=RESOURCES
         + "industrial_production_per_country.csv",
@@ -684,8 +684,8 @@ rule build_transport_demand:
         pop_weighted_energy_totals=RESOURCES
         + "pop_weighted_energy_totals_s{simpl}_{clusters}.csv",
         transport_data=RESOURCES + "transport_data.csv",
-        traffic_data_KFZ="data/emobility/KFZ__count",
-        traffic_data_Pkw="data/emobility/Pkw__count",
+        traffic_data_KFZ="data/bundle-sector/emobility/KFZ__count",
+        traffic_data_Pkw="data/bundle-sector/emobility/Pkw__count",
         temp_air_total=RESOURCES + "temp_air_total_elec_s{simpl}_{clusters}.nc",
     output:
         transport_demand=RESOURCES + "transport_demand_s{simpl}_{clusters}.csv",
@@ -734,7 +734,7 @@ rule prepare_sector_network:
         avail_profile=RESOURCES + "avail_profile_s{simpl}_{clusters}.csv",
         dsm_profile=RESOURCES + "dsm_profile_s{simpl}_{clusters}.csv",
         co2_totals_name=RESOURCES + "co2_totals.csv",
-        co2="data/eea/UNFCCC_v23.csv",
+        co2="data/bundle-sector/eea/UNFCCC_v23.csv",
         biomass_potentials=RESOURCES + "biomass_potentials_s{simpl}_{clusters}.csv",
         heat_profile="data/heat_load_profile_BDEW.csv",
         costs="data/costs_{}.csv".format(config["costs"]["year"])

diff --git a/rules/collect.smk b/rules/collect.smk
@@ -14,12 +14,6 @@ localrules:
     plot_networks,
 
 
-rule all:
-    input:
-        RESULTS + "graphs/costs.pdf",
-    default_target: True
-
-
 rule cluster_networks:
     input:
         expand(RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", **config["scenario"]),

diff --git a/rules/common.smk b/rules/common.smk
@@ -42,7 +42,7 @@ def has_internet_access(url="www.zenodo.org") -> bool:
 def input_eurostat(w):
     # 2016 includes BA, 2017 does not
     report_year = config["energy"]["eurostat_report_year"]
-    return f"data/eurostat-energy_balances-june_{report_year}_edition"
+    return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition"
 
 
 def solved_previous_horizon(wildcards):

diff --git a/rules/postprocess.smk b/rules/postprocess.smk
@@ -106,13 +106,16 @@ rule plot_summary:
         countries=config["countries"],
         planning_horizons=config["scenario"]["planning_horizons"],
         sector_opts=config["scenario"]["sector_opts"],
+        emissions_scope=config["energy"]["emissions"],
+        eurostat_report_year=config["energy"]["eurostat_report_year"],
         plotting=config["plotting"],
         RDIR=RDIR,
     input:
         costs=RESULTS + "csvs/costs.csv",
         energy=RESULTS + "csvs/energy.csv",
         balances=RESULTS + "csvs/supply_energy.csv",
         eurostat=input_eurostat,
+        co2="data/bundle-sector/eea/UNFCCC_v23.csv",
     output:
         costs=RESULTS + "graphs/costs.pdf",
         energy=RESULTS + "graphs/energy.pdf",

diff --git a/rules/retrieve.smk b/rules/retrieve.smk
@@ -27,7 +27,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle",
 
     rule retrieve_databundle:
         output:
-            expand("data/bundle/{file}", file=datafiles),
+            protected(expand("data/bundle/{file}", file=datafiles)),
         log:
             LOGS + "retrieve_databundle.log",
         resources:
@@ -92,7 +92,7 @@ if config["enable"]["retrieve"] and config["enable"].get(
                 static=True,
             ),
         output:
-            RESOURCES + "natura.tiff",
+            protected(RESOURCES + "natura.tiff"),
         log:
             LOGS + "retrieve_natura_raster.log",
         resources:
@@ -106,22 +106,30 @@ if config["enable"]["retrieve"] and config["enable"].get(
     "retrieve_sector_databundle", True
 ):
     datafiles = [
-        "data/eea/UNFCCC_v23.csv",
-        "data/switzerland-sfoe/switzerland-new_format.csv",
-        "data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",
-        "data/myb1-2017-nitro.xls",
-        "data/Industrial_Database.csv",
-        "data/emobility/KFZ__count",
-        "data/emobility/Pkw__count",
-        "data/h2_salt_caverns_GWh_per_sqkm.geojson",
-        directory("data/eurostat-energy_balances-june_2016_edition"),
-        directory("data/eurostat-energy_balances-may_2018_edition"),
-        directory("data/jrc-idees-2015"),
+        "eea/UNFCCC_v23.csv",
+        "switzerland-sfoe/switzerland-new_format.csv",
+        "nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",
+        "myb1-2017-nitro.xls",
+        "Industrial_Database.csv",
+        "emobility/KFZ__count",
+        "emobility/Pkw__count",
+        "h2_salt_caverns_GWh_per_sqkm.geojson",
+    ]
+
+    datafolders = [
+        protected(
+            directory("data/bundle-sector/eurostat-energy_balances-june_2016_edition")
+        ),
+        protected(
+            directory("data/bundle-sector/eurostat-energy_balances-may_2018_edition")
+        ),
+        protected(directory("data/bundle-sector/jrc-idees-2015")),
     ]
 
     rule retrieve_sector_databundle:
         output:
-            *datafiles,
+            protected(expand("data/bundle-sector/{files}", files=datafiles)),
+            *datafolders,
         log:
             LOGS + "retrieve_sector_databundle.log",
         retries: 2
@@ -143,7 +151,9 @@ if config["enable"]["retrieve"] and (
 
     rule retrieve_gas_infrastructure_data:
         output:
-            expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles),
+            protected(
+                expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles)
+            ),
         log:
             LOGS + "retrieve_gas_infrastructure_data.log",
         retries: 2
@@ -187,7 +197,7 @@ if config["enable"]["retrieve"]:
                 static=True,
             ),
         output:
-            "data/shipdensity_global.zip",
+            protected("data/shipdensity_global.zip"),
         log:
             LOGS + "retrieve_ship_raster.log",
         resources:

diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
@@ -435,15 +435,23 @@ def add_heating_capacities_installed_before_baseyear(
 
     # split existing capacities between residential and services
     # proportional to energy demand
+    p_set_sum = n.loads_t.p_set.sum()
     ratio_residential = pd.Series(
         [
             (
-                n.loads_t.p_set.sum()[f"{node} residential rural heat"]
+                p_set_sum[f"{node} residential rural heat"]
                 / (
-                    n.loads_t.p_set.sum()[f"{node} residential rural heat"]
-                    + n.loads_t.p_set.sum()[f"{node} services rural heat"]
+                    p_set_sum[f"{node} residential rural heat"]
+                    + p_set_sum[f"{node} services rural heat"]
                 )
             )
+            # if rural heating demand for one of the nodes doesn't exist,
+            # then columns were dropped before and heating demand share should be 0.0
+            if all(
+                f"{node} {service} rural heat" in p_set_sum.index
+                for service in ["residential", "services"]
+            )
+            else 0.0
             for node in nodal_df.index
         ],
         index=nodal_df.index,

diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py
@@ -13,10 +13,13 @@
 import uuid
 from itertools import product
 
+import country_converter as coco
 import geopandas as gpd
 import pandas as pd
 from packaging.version import Version, parse
 
+cc = coco.CountryConverter()
+
 
 def locate_missing_industrial_sites(df):
     """

diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py
@@ -89,7 +89,7 @@
 def add_custom_powerplants(ppl, custom_powerplants, custom_ppl_query=False):
     if not custom_ppl_query:
         return ppl
-    add_ppls = pd.read_csv(custom_powerplants, index_col=0, dtype={"bus": "str"})
+    add_ppls = pd.read_csv(custom_powerplants, dtype={"bus": "str"})
     if isinstance(custom_ppl_query, str):
         add_ppls.query(custom_ppl_query, inplace=True)
     return pd.concat(

diff --git a/scripts/make_summary.py b/scripts/make_summary.py
@@ -711,5 +711,5 @@ def to_csv(df):
     if snakemake.params.foresight == "myopic":
         cumulative_cost = calculate_cumulative_cost()
         cumulative_cost.to_csv(
-            "results/" + snakemake.params.RDIR + "/csvs/cumulative_cost.csv"
+            "results/" + snakemake.params.RDIR + "csvs/cumulative_cost.csv"
         )