dispatch_validation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dispatch valuation\n",
    "\n",
    "Routines for dispatch validation comprising\n",
    "* Evaluation of imports and exports\n",
    "* Evaluation of storage operation\n",
    "* Evaluation of dispatch per energy carrier against historical one\n",
    "* Evaluation of single dispatch situations\n",
    "* Evaluation of energy not served and scarcity events"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Package imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import OrderedDict\n",
    "from pommesevaluation.dispatch_validation import (\n",
    "    load_entsoe_german_generation_data, plot_imports_and_exports, read_and_reshape_historical_im_ex\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read in and filter for electrical bus results for Germany"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "simulation_year = 2017\n",
    "path_results = \"./model_results/pommesdispatch/\"\n",
    "path_plots = \"./plots/\"\n",
    "path_historical_production = \"./data/production/\"\n",
    "eeg_clusters_per_technology = 20\n",
    "\n",
    "model_file_name = (\n",
    "    f\"dispatch_LP_start-{simulation_year}-01-01_364-days_simple_complete_\"\n",
    "    f\"{eeg_clusters_per_technology}_res-clusters_production.csv\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "buses_el = pd.read_csv(path_results + model_file_name, index_col=0)\n",
    "if simulation_year < 2022:\n",
    "    historical_production = load_entsoe_german_generation_data(\n",
    "        path=f\"{path_historical_production}\", year=simulation_year\n",
    "    )\n",
    "\n",
    "# Filter generation, exports and imports for Germany\n",
    "bus_DE = buses_el[[col for col in buses_el.columns if \"DE\" in col]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate exports & imports\n",
    "* Calculate overall exports and imports and net export\n",
    "* Plot exports / imports by country\n",
    "* Evaluate against historical exports and imports\n",
    "\n",
    "### Calculate and plot overall imports and exports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter imports and exports and calculate overall and net imports & exports\n",
    "im_ex_DE = bus_DE[[col for col in bus_DE.columns if \"link_\" in col]].copy()\n",
    "export_links = [col for col in im_ex_DE if \"('DE_link_\" in col]\n",
    "import_links = [col for col in im_ex_DE if \"DE_bus_el')\" in col]\n",
    "im_ex_DE[\"overall_exports\"] = im_ex_DE[export_links].sum(axis=1)\n",
    "im_ex_DE[\"overall_imports\"] = -im_ex_DE[import_links].sum(axis=1)\n",
    "im_ex_DE[\"net_export\"] = im_ex_DE[\"overall_exports\"] + im_ex_DE[\"overall_imports\"]\n",
    "im_ex_DE.index = pd.to_datetime(im_ex_DE.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(15, 5)) \n",
    "\n",
    "_ = im_ex_DE[[\"overall_imports\", \"overall_exports\"]].plot(ax=ax, color=[\"red\", \"blue\"], alpha=0.3)\n",
    "_ = im_ex_DE[[\"net_export\"]].plot(ax=ax, color=\"black\")\n",
    "_ = ax.set_xlabel(\"Time\")\n",
    "_ = ax.set_ylabel(\"Energy in MWh\")\n",
    "_ = plt.legend(loc=\"upper right\")\n",
    "_ = plt.tight_layout()\n",
    "_ = plt.savefig(f\"{path_plots}overall_im_and_exports_{simulation_year}.png\", dpi=300)\n",
    "\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate imports and exports patterns for certain time frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "im_ex_plot = im_ex_DE.copy()\n",
    "im_ex_plot[import_links] *= -1\n",
    "imports = {\n",
    "    key: f\"import_{key[3:5]}\" \n",
    "    for key in import_links \n",
    "    if \"DK\" not in key and \"SE\" not in key\n",
    "}\n",
    "for key in import_links:\n",
    "    if \"DK\" in key or \"SE\" in key:\n",
    "        imports[key] = f\"import_{key[3:6]}\"\n",
    "\n",
    "exports = {\n",
    "    key: f\"export_{key[11:13]}\" \n",
    "    for key in export_links\n",
    "    if \"DK\" not in key and \"SE\" not in key\n",
    "}\n",
    "for key in export_links:\n",
    "    if \"DK\" in key or \"SE\" in key:\n",
    "        exports[key] = f\"export_{key[11:14]}\"\n",
    "\n",
    "im_ex_plot.rename(columns=imports, inplace=True)\n",
    "im_ex_plot.rename(columns=exports, inplace=True)\n",
    "im_ex_plot.drop(\n",
    "    columns=[\n",
    "        col for col in im_ex_plot.columns \n",
    "        if col not in imports.values() and col not in exports.values()\n",
    "    ], \n",
    "    inplace=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "country_colors = {\n",
    "    \"FR\": \"blue\",\n",
    "    \"CH\": \"red\",\n",
    "    \"BE\": \"black\",\n",
    "    \"CZ\": \"gray\",\n",
    "    \"DK1\": \"lightblue\",\n",
    "    \"DK2\": \"turquoise\",\n",
    "    \"PL\": \"dimgray\",\n",
    "    \"NL\": \"orange\",\n",
    "    \"NO\": \"green\",\n",
    "    \"SE4\": \"yellow\",\n",
    "    \"AT\": \"purple\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colors_im = {\n",
    "    f\"import_{key}\": val \n",
    "    for key, val in country_colors.items() \n",
    "    for col in im_ex_plot.columns if key in col\n",
    "}\n",
    "colors_ex = {\n",
    "    f\"export_{key}\": val \n",
    "    for key, val in country_colors.items() \n",
    "    for col in im_ex_plot.columns if key in col\n",
    "}\n",
    "country_colors_im_ex = {**colors_im, **colors_ex}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove exchange with Austria before market splitting in 2018\n",
    "if simulation_year <= 2018:\n",
    "    im_ex_plot.drop(columns=[\"export_AT\", \"import_AT\"], inplace=True)\n",
    "    country_colors.pop(\"AT\")\n",
    "    country_colors_im_ex.pop(\"export_AT\")\n",
    "    country_colors_im_ex.pop(\"import_AT\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_imports_and_exports(\n",
    "    im_ex_plot, \n",
    "    country_colors_im_ex, \n",
    "    start=f\"{simulation_year}-01-01 00:00\", \n",
    "    end=f\"{simulation_year}-01-07 23:00\",\n",
    "    save=True,\n",
    "    path_plots=path_plots,\n",
    "    file_name=f\"imports_and_exports_{simulation_year}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate net exports / imports only\n",
    "* Calculate sums for net exports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for country in country_colors.keys():\n",
    "    im_ex_plot[f\"net_export_{country}\"] = (\n",
    "        im_ex_plot[f\"export_{country}\"]\n",
    "        + im_ex_plot[f\"import_{country}\"]\n",
    "    )\n",
    "im_ex_plot.drop(\n",
    "    columns=[\n",
    "        col for col in im_ex_plot.columns\n",
    "        if \"net_export\" not in col\n",
    "    ],\n",
    "    inplace=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in im_ex_plot.columns:\n",
    "    im_ex_plot[f\"{col}_pos\"] = np.where(im_ex_plot[col] >= 0, im_ex_plot[col], 0)\n",
    "    im_ex_plot[f\"{col}_neg\"] = np.where(im_ex_plot[col] < 0, im_ex_plot[col], 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "im_ex_plot = im_ex_plot.drop(\n",
    "    columns=[\n",
    "        col for col in im_ex_plot\n",
    "        if \"pos\" not in col and \"neg\" not in col\n",
    "    ]\n",
    ")\n",
    "im_ex_overall = im_ex_plot.copy()\n",
    "im_ex_overall[\"overall_net_export\"] = im_ex_overall.sum(axis=1)\n",
    "im_ex_overall[\"overall_net_export_pos\"] = np.where(\n",
    "    im_ex_overall[\"overall_net_export\"] >= 0,   \n",
    "    im_ex_overall[\"overall_net_export\"], \n",
    "    0\n",
    ")\n",
    "im_ex_overall[\"overall_net_export_neg\"] = np.where(\n",
    "    im_ex_overall[\"overall_net_export\"] < 0, \n",
    "    im_ex_overall[\"overall_net_export\"], \n",
    "    0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "country_colors_net_exports_pos = {\n",
    "    f\"net_export_{key}_pos\": val \n",
    "    for key, val in country_colors.items() \n",
    "    for col in im_ex_plot.columns if key in col\n",
    "}\n",
    "country_colors_net_exports_neg = {\n",
    "    f\"net_export_{key}_neg\": val \n",
    "    for key, val in country_colors.items() \n",
    "    for col in im_ex_plot.columns if key in col\n",
    "} \n",
    "country_colors_net_exports = {\n",
    "    **country_colors_net_exports_pos, **country_colors_net_exports_neg\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_imports_and_exports(\n",
    "    im_ex_plot, \n",
    "    country_colors_net_exports, \n",
    "    start=f\"{simulation_year}-01-01 00:00\", \n",
    "    end=f\"{simulation_year}-01-07 23:00\",\n",
    "    save=True,\n",
    "    path_plots=path_plots,\n",
    "    file_name=f\"net_imports_and_exports_{simulation_year}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare modelled with historical exports and imports\n",
    "* Read in and preprocess historical data\n",
    "* Compare annual sums against each other"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if simulation_year < 2022:\n",
    "    if not simulation_year == 2018:\n",
    "        historical_im_ex = read_and_reshape_historical_im_ex(path_historical_production, simulation_year)\n",
    "    else:\n",
    "        historical_im_ex = pd.concat([\n",
    "            read_and_reshape_historical_im_ex(path_historical_production, simulation_year),\n",
    "            read_and_reshape_historical_im_ex(\n",
    "                path_historical_production, \n",
    "                simulation_year, \n",
    "                file_name=f\"Kommerzieller_Au_enhandel_{simulation_year}10010000_{simulation_year}12312359.xlsx\"\n",
    "            ),\n",
    "        ])\n",
    "else:\n",
    "    historical_im_ex = read_and_reshape_historical_im_ex(path_historical_production, 2017)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "overall_net_exports = pd.DataFrame(index=historical_im_ex.columns)\n",
    "overall_net_exports[\"model\"] = im_ex_plot.sum()\n",
    "\n",
    "if simulation_year < 2022:\n",
    "    # Replace nan / string values before proceeding\n",
    "    historical_im_ex.replace(\"----\", 0, inplace=True)\n",
    "    historical_im_ex = historical_im_ex.astype(\"float64\")\n",
    "\n",
    "    overall_net_exports[\"historical\"] = historical_im_ex.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(15,5))\n",
    "_ = overall_net_exports.plot(kind=\"bar\", ax=ax)\n",
    "_ = plt.axhline(y=0, color='gray', linestyle='-.', linewidth=.4)\n",
    "_ = plt.title(\"Comparison of exports (pos) and imports (neg)\")\n",
    "_ = plt.tight_layout()\n",
    "_ = plt.savefig(f\"{path_plots}comparison_exports_imports_{simulation_year}.png\", dpi=300)\n",
    "\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate saldo of imports / exports\n",
    "countries_modelled = [\"DK1\", \"DK2\", \"NL\", \"CH\", \"CZ\", \"FR\", \"SE4\", \"PL\"]\n",
    "for country in countries_modelled:\n",
    "    overall_net_exports.loc[country] = (\n",
    "        overall_net_exports.loc[f\"net_export_{country}_pos\"] \n",
    "        + overall_net_exports.loc[f\"net_export_{country}_neg\"]\n",
    "    )\n",
    "\n",
    "overall_net_exports = overall_net_exports.loc[countries_modelled]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(15,5))\n",
    "_ = overall_net_exports.plot(kind=\"bar\", ax=ax)\n",
    "_ = plt.axhline(y=0, color='gray', linestyle='-.', linewidth=.4)\n",
    "_ = plt.title(\"Comparison of net exports (pos) and imports (neg)\")\n",
    "_ = plt.tight_layout()\n",
    "_ = plt.savefig(f\"{path_plots}comparison_exports_imports_saldo_{simulation_year}.png\", dpi=300)\n",
    "\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate storage operation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "storage_DE = bus_DE[[col for col in bus_DE.columns if \"storage\" in col]]\n",
    "storage_DE_renamed = storage_DE.rename(columns={\n",
    "    \"(('DE_bus_el', 'DE_storage_el_PHS'), 'flow')\": \"storage_in\",\n",
    "    \"(('DE_storage_el_PHS', 'DE_bus_el'), 'flow')\": \"storage_out\"\n",
    "})\n",
    "storage_DE_renamed[\"storage_in\"] *= -1\n",
    "storage_DE_renamed[\"net_storage\"] = storage_DE_renamed[\"storage_out\"] + storage_DE_renamed[\"storage_in\"]\n",
    "storage_DE_renamed.index = pd.to_datetime(storage_DE_renamed.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(15, 5))    \n",
    "\n",
    "\n",
    "_ = ax.fill_between(\n",
    "    storage_DE_renamed.index, 0, storage_DE_renamed.storage_in,\n",
    "    step='post',\n",
    "    facecolor='red',\n",
    "    label=\"storage_in\",\n",
    "    alpha=1\n",
    ")\n",
    "_ = ax.fill_between(\n",
    "    storage_DE_renamed.index, 0, storage_DE_renamed.storage_out,\n",
    "    step='post',\n",
    "    facecolor='blue',\n",
    "    label=\"storage_out\",\n",
    "    alpha=1\n",
    ")\n",
    "\n",
    "_ = plt.axhline(y=0, color='gray', linestyle='-.', linewidth=.4)\n",
    "_ = plt.title(\"Storage pattern\")\n",
    "_ = plt.tight_layout()\n",
    "ax.set_xlabel(\"Time\")\n",
    "ax.set_ylabel(\"Energy in MWh\")\n",
    "_ = plt.legend()\n",
    "_ = plt.tight_layout\n",
    "_ = plt.savefig(f\"{path_plots}storage_pattern_{simulation_year}.png\", dpi=300)\n",
    "\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate power generation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Aggregate generation by fuel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Aggregate all generation units\n",
    "power_generation_DE = bus_DE[[\n",
    "    col for col in bus_DE.columns \n",
    "    if col not in im_ex_DE.columns\n",
    "    and col not in storage_DE.columns\n",
    "]]\n",
    "\n",
    "# Drop demand, power prices and shortage\n",
    "power_generation_DE = power_generation_DE.drop(\n",
    "    columns=[\n",
    "        \"(('DE_bus_el', 'DE_sink_el_load'), 'flow')\", \n",
    "        \"(('DE_bus_el', 'None'), 'duals')\",\n",
    "        \"(('DE_source_el_shortage', 'DE_bus_el'), 'flow')\",\n",
    "    ]\n",
    ")\n",
    "\n",
    "power_generation_DE[\"overall_generation\"] = power_generation_DE.sum(axis=1)\n",
    "power_generation_DE.index = pd.to_datetime(power_generation_DE.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fuel_dict = {\n",
    "    'ROR': 'Wasser',\n",
    "    'biomass': 'Biomasse',\n",
    "    'biomassEEG': 'Biomasse',\n",
    "    'landfillgas': 'Deponiegas',\n",
    "    'geothermal': 'Geothermie',\n",
    "    'minegas': 'Grubengas',\n",
    "    'larga': 'Klärgas',\n",
    "    'windonshore': 'Windenergie an Land',\n",
    "    'windoffshore': 'Windenergie auf See',\n",
    "    'solarPV': 'Solare Strahlungsenergie',\n",
    "    'uranium': 'Kernenergie',\n",
    "    'lignite': 'Braunkohle',\n",
    "    'hardcoal': 'Steinkohle',\n",
    "    'waste': 'Abfall',\n",
    "    'natgas': 'Erdgas',\n",
    "    'otherfossil': 'Andere fossile',\n",
    "    'mixedfuels': 'Mehrere fossile',\n",
    "    'oil': 'Heizöl',\n",
    "    'hydrogen': 'Wasserstoff',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colors = {\n",
    "    'solarPV': '#fcb001',\n",
    "    'windonshore': '#82cafc',\n",
    "    'windoffshore': '#0504aa',\n",
    "    'uranium': '#e50000',\n",
    "    'lignite': '#7f2b0a',\n",
    "    'otherfossil': '#d8dcd6',\n",
    "    'hardcoal': '#000000',\n",
    "    'waste': '#c04e01',\n",
    "    'mixedfuels': '#a57e52',\n",
    "    'biomass': '#15b01a',\n",
    "    'geothermal': '#ff474c',\n",
    "    'otherres': '#06c2ac',\n",
    "    'minegas': '#650021',\n",
    "    'natgas': '#929591',\n",
    "    'oil': '#aaa662',\n",
    "    'ROR': '#c79fef',\n",
    "    'storage_el_out' : 'darkblue',\n",
    "    'hydrogen': '#6fa8dc',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Group outputs by energy carrier\n",
    "energy_sources_dict = OrderedDict()\n",
    "\n",
    "for fuel in fuel_dict.keys():\n",
    "    energy_sources_dict[fuel] = [\n",
    "        entry for entry in power_generation_DE.columns.values if fuel in entry\n",
    "    ]   \n",
    "    \n",
    "# Store the aggregated production results per energy source\n",
    "generation = pd.DataFrame()\n",
    "for key, val in energy_sources_dict.items():\n",
    "    generation[key] = power_generation_DE[val].sum(axis = 1)\n",
    "\n",
    "# Aggregate\n",
    "generation[\"biomass\"] = generation[\"biomass\"] + generation[\"biomassEEG\"]\n",
    "generation[\"otherres\"] = (\n",
    "    generation[\"landfillgas\"]\n",
    "    + generation[\"larga\"]\n",
    ")\n",
    "\n",
    "generation.drop(\n",
    "    columns=[\n",
    "        col for col in generation.columns\n",
    "        if col not in colors.keys()\n",
    "    ],\n",
    "    inplace=True\n",
    ")\n",
    "\n",
    "generation[\"storage_el_out\"] = storage_DE_renamed[\"storage_out\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize generation for certain time frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(15, 5))\n",
    "_ = generation.iloc[4000:4369].plot(ax=ax, kind=\"area\", color=colors)\n",
    "_ = ax.set_xlabel(\"Time\")\n",
    "_ = ax.set_ylabel(\"Energy produced [MWh/h]\")\n",
    "_ = plt.legend(bbox_to_anchor=[1.02, 1.05])\n",
    "_ = plt.tight_layout()\n",
    "_ = plt.savefig(f\"{path_plots}generation_{simulation_year}.png\", dpi=300)\n",
    "\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare summed generation againts historical one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "overall_generation = pd.DataFrame(index=generation.columns)\n",
    "overall_generation[\"model\"] = generation.sum()\n",
    "if simulation_year < 2022:\n",
    "    overall_generation[\"historical\"] = historical_production.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(15,5))\n",
    "_ = overall_generation.plot(kind=\"bar\", ax=ax)\n",
    "_ = plt.title(\"Comparison of production by fuel\")\n",
    "_ = plt.tight_layout()\n",
    "_ = plt.savefig(f\"{path_plots}production_by_fuel_{simulation_year}.png\", dpi=300)\n",
    "\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(power_generation_DE.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyze dedicated supply situations\n",
    "Analyze the supply situation for certain time steps / time frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time_step = f\"{simulation_year}-01-01 00:00:00\"\n",
    "end_time_step = f\"{simulation_year}-12-31 23:00:00\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "overall_situation = pd.concat(\n",
    "    [\n",
    "        storage_DE_renamed.loc[start_time_step:end_time_step, \"storage_in\"],\n",
    "        generation.loc[start_time_step:end_time_step],\n",
    "        im_ex_overall.loc[start_time_step:end_time_step, [\"overall_net_export_pos\", \"overall_net_export_neg\"]]\n",
    "    ],\n",
    "    axis = 1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "overall_situation.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time_step = f\"{simulation_year}-04-30 00:00:00\"\n",
    "end_time_step = f\"{simulation_year}-04-30 23:00:00\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "overall_situation_slice = overall_situation.loc[start_time_step: end_time_step].round(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "overall_situtation_colors = {\n",
    "    **colors,\n",
    "    \"storage_in\": \"darkblue\", \n",
    "    \"overall_net_export_pos\": \"#ffefef\",\n",
    "    \"overall_net_export_neg\": \"#ffefef\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(15, 10))\n",
    "_ = overall_situation_slice.plot(ax=ax, kind=\"area\", color=overall_situtation_colors)\n",
    "_ = ax.set_xlabel(\"Time\")\n",
    "_ = ax.set_ylabel(\"Energy [MWh/h]\")\n",
    "_ = plt.title(f\"Dispatch situation between {start_time_step} and {end_time_step}\")\n",
    "_ = plt.legend(bbox_to_anchor=[1.02, 1.05])\n",
    "_ = plt.tight_layout()\n",
    "_ = plt.savefig(f\"{path_plots}excess_situation_{simulation_year}.png\", dpi=300)\n",
    "\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "overall_situation_slice.loc[f\"{simulation_year}-04-30 13:00\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyze scarcity and excess generation situations\n",
    "\n",
    "### Calculate domestic net demand (after storage and exports / imports)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "demand_DE = bus_DE[[\"(('DE_bus_el', 'DE_sink_el_load'), 'flow')\"]].rename(\n",
    "    columns={\n",
    "        \"(('DE_bus_el', 'DE_sink_el_load'), 'flow')\": \"domestic_demand\"\n",
    "    }\n",
    ")\n",
    "demand_DE.index = pd.to_datetime(demand_DE.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "demand_DE.loc[f\"{simulation_year}-04-30 13:00\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Contrast demand and generation for real scarcity situations (energy not served)\n",
    "* Calculate generation after net storage and net exports and compare with domestic demand\n",
    "* Identify the difference, i.e. shortages\n",
    "* Identify maximum shortage value and time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "balance = power_generation_DE[[\"overall_generation\"]].copy()\n",
    "balance[\"net_export\"] = im_ex_DE[\"net_export\"]\n",
    "balance[\"net_storage\"] = storage_DE_renamed[\"net_storage\"]\n",
    "balance[\"gen_ex_stor\"] = balance[\"overall_generation\"] + balance[\"net_storage\"] - balance[\"net_export\"]\n",
    "balance[\"demand\"] = demand_DE[\"domestic_demand\"]\n",
    "balance[\"delta\"] = balance[\"gen_ex_stor\"] - balance[\"demand\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "balance[\"delta\"].loc[f\"{simulation_year}-01-23 12:00\":f\"{simulation_year}-01-25\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Identify scarcity situations\n",
    "numeric_zero = 1e5\n",
    "balance[\"delta\"].loc[balance[\"delta\"] < -numeric_zero]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bus_DE[\"(('DE_source_el_shortage', 'DE_bus_el'), 'flow')\"].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bus_DE[\"(('DE_source_el_shortage', 'DE_bus_el'), 'flow')\"].idxmax()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Excess situations occuring in AT; DE has no excess sink\n",
    "excess_sinks = buses_el.loc[:,[col for col in buses_el.columns if \"excess\" in col]]\n",
    "excess_sinks.loc[excess_sinks[\"(('AT_bus_el', 'AT_sink_el_excess'), 'flow')\"] > 0.01]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate scarcity situations where demand is met\n",
    "Evaluate the dispatch of artificial scarcity units which are introduced to enable a market clearing in order to prevent energy being not served"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "512px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}