Update calculations of retrofit costs

envs/environment.yaml

@@ -11,7 +11,7 @@ dependencies:
 - pip
 
 - atlite>=0.2.9
-- dask
+- dask<=2023.9.1
 
   # Dependencies of the workflow itself
 - xlrd

scripts/build_retro_cost.py

@@ -198,12 +198,13 @@ def prepare_building_stock_data():
         }
     )
 
+    building_data["country_code"] = building_data["country"].map(country_iso_dic)
+
     # heated floor area ----------------------------------------------------------
     area = building_data[
-        (building_data.type == "Heated area [Mm²]")
-        & (building_data.subsector != "Total")
+        (building_data.type == "Heated area [Mm²]") & (building_data.detail != "Total")
     ]
-    area_tot = area.groupby(["country", "sector"]).sum()
+    area_tot = area[["country", "sector", "value"]].groupby(["country", "sector"]).sum()
     area = pd.concat(
         [
             area,
@@ -223,7 +224,7 @@ def prepare_building_stock_data():
         usecols=[0, 1, 2, 3],
         encoding="ISO-8859-1",
     )
-    area_tot = area_tot.append(area_missing.unstack(level=-1).dropna().stack())
+    area_tot = pd.concat([area_tot, area_missing.unstack(level=-1).dropna().stack()])
     area_tot = area_tot.loc[~area_tot.index.duplicated(keep="last")]
 
     # for still missing countries calculate floor area by population size
@@ -246,7 +247,7 @@ def prepare_building_stock_data():
         averaged_data.index = index
         averaged_data["estimated"] = 1
         if ct not in area_tot.index.levels[0]:
-            area_tot = area_tot.append(averaged_data, sort=True)
+            area_tot = pd.concat([area_tot, averaged_data], sort=True)
         else:
             area_tot.loc[averaged_data.index] = averaged_data
 
@@ -272,7 +273,7 @@ def prepare_building_stock_data():
             ][x["bage"]].iloc[0],
             axis=1,
         )
-        data_PL_final = data_PL_final.append(data_PL)
+        data_PL_final = pd.concat([data_PL_final, data_PL])
 
     u_values = pd.concat([u_values, data_PL_final]).reset_index(drop=True)
 
@@ -943,7 +944,8 @@ def sample_dE_costs_area(
             .rename(index=rename_sectors, level=2)
             .reset_index()
         )
-        .rename(columns={"country": "country_code"})
+        # if uncommented, leads to the second `country_code` column
+        # .rename(columns={"country": "country_code"})
         .set_index(["country_code", "subsector", "bage"])
     )
 
@@ -956,13 +958,14 @@ def sample_dE_costs_area(
     )
 
     # map missing countries
-    for ct in countries.difference(cost_dE.index.levels[0]):
+    for ct in set(countries).difference(cost_dE.index.levels[0]):
         averaged_data = (
             cost_dE.reindex(index=map_for_missings[ct], level=0)
-            .mean(level=1)
+            .groupby(level=1)
+            .mean()
             .set_index(pd.MultiIndex.from_product([[ct], cost_dE.index.levels[1]]))
         )
-        cost_dE = cost_dE.append(averaged_data)
+        cost_dE = pd.concat([cost_dE, averaged_data])
 
     # weights costs after construction index
     if construction_index:
@@ -979,24 +982,23 @@ def sample_dE_costs_area(
     # drop not considered countries
     cost_dE = cost_dE.reindex(countries, level=0)
     # get share of residential and service floor area
-    sec_w = area_tot.value / area_tot.value.groupby(level=0).sum()
+    sec_w = area_tot.div(area_tot.groupby(level=0).transform("sum"))
     # get the total cost-energy-savings weight by sector area
     tot = (
-        cost_dE.mul(sec_w, axis=0)
-        .groupby(level="country_code")
+        # sec_w has columns "estimated" and "value"
+        cost_dE.mul(sec_w.value, axis=0)
+        # for some reasons names of the levels were lost somewhere
+        # .groupby(level="country_code")
+        .groupby(level=0)
         .sum()
-        .set_index(
-            pd.MultiIndex.from_product(
-                [cost_dE.index.unique(level="country_code"), ["tot"]]
-            )
-        )
+        .set_index(pd.MultiIndex.from_product([cost_dE.index.unique(level=0), ["tot"]]))
     )
-    cost_dE = cost_dE.append(tot).unstack().stack()
+    cost_dE = pd.concat([cost_dE, tot]).unstack().stack()
 
-    summed_area = pd.DataFrame(area_tot.groupby("country").sum()).set_index(
-        pd.MultiIndex.from_product([area_tot.index.unique(level="country"), ["tot"]])
+    summed_area = pd.DataFrame(area_tot.groupby(level=0).sum()).set_index(
+        pd.MultiIndex.from_product([area_tot.index.unique(level=0), ["tot"]])
     )
-    area_tot = area_tot.append(summed_area).unstack().stack()
+    area_tot = pd.concat([area_tot, summed_area]).unstack().stack()
 
     cost_per_saving = cost_dE["cost"] / (
         1 - cost_dE["dE"]

scripts/prepare_sector_network.py

@@ -1954,7 +1954,7 @@ def add_heat(n, costs):
         # demand 'dE' [per unit of original heat demand] for each country and
         # different retrofitting strengths [additional insulation thickness in m]
         retro_data = pd.read_csv(
-            snakemake.input.retro_cost_energy,
+            snakemake.input.retro_cost,
             index_col=[0, 1],
             skipinitialspace=True,
             header=[0, 1],