Update energy balance for residential based on new Eurostat data

rules/build_sector.smk

@@ -293,6 +293,7 @@ rule build_energy_totals:
         idees="data/bundle-sector/jrc-idees-2015",
         district_heat_share="data/district_heat_share.csv",
         eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
+        eurostat_households="data/eurostat/eurostat-household_energy_balances-february_2024.csv",
     output:
         energy_name=resources("energy_totals.csv"),
         co2_name=resources("co2_totals.csv"),

rules/retrieve.smk

@@ -148,6 +148,7 @@ if config["enable"]["retrieve"] and config["enable"].get(
     rule retrieve_eurostat_data:
         output:
             directory("data/eurostat/eurostat-energy_balances-april_2023_edition"),
+            "data/eurostat/eurostat-household_energy_balances-february_2024.csv",
         log:
             "logs/retrieve_eurostat_data.log",
         retries: 2

scripts/build_energy_totals.py

@@ -943,6 +943,60 @@ def rescale_idees_from_eurostat(
     return energy
 
 
+def update_residential_from_eurostat(energy):
+    """
+    Updates energy balances for residential from disaggregated data from
+    Eurostat.
+    """
+    # Read disaggregated Eurostat's data
+    fn = snakemake.input.eurostat_households
+    eurostat_data = pd.read_csv(fn)
+
+    # Column mapping for energy type
+    nrg_type = {
+        "total residential": "FC_OTH_HH_E",
+        "total residential space": "FC_OTH_HH_E_SH",
+        "total residential water": "FC_OTH_HH_E_WH",
+        "total residential cooking": "FC_OTH_HH_E_CK",
+    }
+
+    # Make temporary copy of energy_totals
+    energy_totals = energy.copy().reset_index()
+
+    for nrg_name, code in nrg_type.items():
+        # Select energy balance type
+        nrg_data = eurostat_data.query("nrg_bal in @code").copy()
+        # Rename columns
+        nrg_data.rename(
+            columns={"geo": "country", "TIME_PERIOD": "year", "OBS_VALUE": nrg_name},
+            inplace=True,
+        )
+        # Convert TJ to TWh
+        nrg_data[nrg_name] = nrg_data[nrg_name] / 3.6e3
+        # Select value, country, year columns
+        nrg_data = nrg_data[["country", "year", nrg_name]]
+        # To update energy data with Eurostat households data
+        # 1) Merge the two DataFrames on 'year' and 'country'
+        merged_df = energy_totals.merge(
+            nrg_data,
+            on=["year", "country"],
+            suffixes=("_energy_totals", "_nrg_data"),
+            how="left",
+        )
+        # 2) Update the 'nrg_name' column in energy with the values from nrg_data
+        energy_totals[nrg_name] = merged_df[f"{nrg_name}_nrg_data"].combine_first(
+            merged_df[f"{nrg_name}_energy_totals"]
+        )
+
+    # Set indexes back
+    energy_totals.set_index(["country", "year"], inplace=True)
+    logger.info(
+        "Updated energy balances for residential using disaggregate final energy consumption data in Households from Eurostat"
+    )
+
+    return energy_totals
+
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
@@ -976,6 +1030,8 @@ def rescale_idees_from_eurostat(
     logger.info("Extrapolate IDEES data based on eurostat for years 2015-2021.")
     energy = rescale_idees_from_eurostat(idees_countries, energy, eurostat)
 
+    energy = update_residential_from_eurostat(energy)
+
     energy.to_csv(snakemake.output.energy_name)
 
     # use rescaled idees data to calculate district heat share

scripts/retrieve_eurostat_data.py

@@ -7,7 +7,9 @@
 """
 
 
+import gzip
 import logging
+import shutil
 import zipfile
 from pathlib import Path
 
@@ -41,3 +43,22 @@
         zip_ref.extractall(to_fn)
 
     logger.info(f"Eurostat data available in '{to_fn}'.")
+
+    url_eurostat_household = "https://ec.europa.eu/eurostat/api/dissemination/sdmx/3.0/data/dataflow/ESTAT/nrg_d_hhq/1.0/*.*.*.*.*?c[freq]=A&c[nrg_bal]=FC_OTH_HH_E,FC_OTH_HH_E_SH,FC_OTH_HH_E_WH,FC_OTH_HH_E_CK&c[siec]=TOTAL&c[unit]=TJ&c[geo]=EU27_2020,EA20,BE,BG,CZ,DK,DE,EE,IE,EL,ES,FR,HR,IT,CY,LV,LT,LU,HU,MT,NL,AT,PL,PT,RO,SI,SK,FI,SE,NO,UK,BA,MD,MK,AL,RS,UA,XK,GE&compress=true&format=csvdata&formatVersion=2.0&c[time]=2021,2020,2019,2018,2017,2016,2015,2014,2013,2012,2011,2010"
+    tarball_fn = Path(f"{rootpath}/data/eurostat/eurostat_household.gz")
+    to_fn = Path(
+        f"{rootpath}/data/eurostat/eurostat-household_energy_balances-february_2024.csv"
+    )
+
+    logger.info(
+        f"Downloading Eurostats' disaggregated household energy balances data from '{url_eurostat_household}'."
+    )
+    progress_retrieve(url_eurostat_household, tarball_fn, disable=disable_progress)
+
+    logger.info("Extracting Eurostat's disaggregated household energy balance data.")
+    with gzip.open(tarball_fn, "rb") as f_in, open(to_fn, "wb") as f_out:
+        shutil.copyfileobj(f_in, f_out)
+
+    logger.info(
+        f"Eurostat's disaggregated household energy balance data available in '{to_fn}'."
+    )