Merge branch 'master' into implement-floating

.github/workflows/ci.yaml

@@ -19,7 +19,7 @@ on:
   - cron: "0 5 * * TUE"
 
 env:
-  DATA_CACHE_NUMBER: 1
+  DATA_CACHE_NUMBER: 2
 
 jobs:
   build:

.gitignore

@@ -37,18 +37,16 @@ dconf
 /data/links_p_nom.csv
 /data/*totals.csv
 /data/biomass*
-/data/bundle-sector/emobility/
-/data/bundle-sector/eea*
-/data/bundle-sector/jrc*
+/data/bundle/emobility/
+/data/bundle/eea*
+/data/bundle/jrc*
 /data/heating/
-/data/bundle-sector/eurostat*
+/data/bundle/eurostat*
 /data/odyssee/
 /data/transport_data.csv
-/data/bundle-sector/switzerland*
 /data/.nfs*
-/data/bundle-sector/Industrial_Database.csv
-/data/retro/tabula-calculator-calcsetbuilding.csv
-/data/bundle-sector/nuts*
+/data/retro/*
+/data/bundle/nuts*
 data/gas_network/scigrid-gas/
 data/costs_*.csv
 

.reuse/dep5

@@ -30,7 +30,3 @@ License: CC0-1.0
 Files: borg-it
 Copyright: 2017-2024 The PyPSA-Eur Authors
 License: CC0-1.0
-
-Files: graphics/*
-Copyright: 2017-2024 The PyPSA-Eur Authors
-License: CC-BY-4.0

README.md

@@ -80,7 +80,7 @@ all greenhouse gas emitters except waste management and land use.
 This diagram gives an overview of the sectors and the links between
 them:
 
-![sector diagram](graphics/multisector_figure.png)
+![sector diagram](doc/img/multisector_figure.png)
 
 Each of these sectors is built up on the transmission network nodes
 from [PyPSA-Eur](https://github.com/PyPSA/pypsa-eur):

config/config.default.yaml

@@ -69,12 +69,9 @@ enable:
   retrieve: auto
   prepare_links_p_nom: false
   retrieve_databundle: true
-  retrieve_sector_databundle: true
   retrieve_cost_data: true
   build_cutout: false
   retrieve_cutout: true
-  build_natura_raster: false
-  retrieve_natura_raster: true
   custom_busmap: false
   drop_leap_day: true
 

config/config.entsoe-all.yaml

@@ -38,6 +38,5 @@ lines:
 enable:
   retrieve: true
   retrieve_databundle: true
-  retrieve_sector_databundle: false
   retrieve_cost_data: true
   retrieve_cutout: true

data/ch_cantons.csv

@@ -0,0 +1,27 @@
+Canton,HASC,NUTS
+Aargau,CH.AG,CH033
+Appenzell Inner Rhodes,CH.AI,CH054
+Appenzell Outer Rhodes,CH.AR,CH053
+Basel-Landschaft,CH.BL,CH032
+Basel-Stadt,CH.BS,CH031
+Bern,CH.BE,CH021
+Fribourg,CH.FR,CH022
+Geneva,CH.GE,CH013
+Glarus,CH.GL,CH051
+Graubünden,CH.GR,CH056
+Jura,CH.JU,CH025
+Lucerne,CH.LU,CH061
+Neuchâtel,CH.NE,CH024
+Nidwalden,CH.NW,CH065
+Obwalden,CH.OW,CH064
+Sankt Gallen,CH.SG,CH055
+Schaffhausen,CH.SH,CH052
+Schwyz,CH.SZ,CH063
+Solothurn,CH.SO,CH023
+Thurgau,CH.TG,CH057
+Ticino,CH.TI,CH07
+Uri,CH.UR,CH062
+Valais,CH.VS,CH012
+Vaud,CH.VD,CH011
+Zug,CH.ZG,CH066
+Zurich,CH.ZH,CH04

data/hydro_capacities.csv

@@ -0,0 +1,31 @@
+Country,p_nom_discharge[GW],p_nom_store[GW],E_store[TWh],InflowHourlyAvg[GWh]
+AT,13.08,3.8,3.2,4.02
+BE,1.42,1.31,0,0.04
+BA,2.05,0.62,2.5,0.71
+BG,3.13,0.86,4,0.53
+HR,2,0.61,2.8,0.57
+CZ,2.21,0.68,1.5,0.24
+DK,0.01,0,0,0
+EE,0.01,0,0,0
+FI,3.2,0,5.5,1.59
+FR,25.37,6.99,9.8,7.82
+DE,11.26,6.8,0.3,1.93
+GB,4.43,2.74,0,0.46
+GR,3.24,0.7,2.3,0.26
+HU,0.06,0,0.1,0.02
+IE,0.53,0.29,0,0.08
+IT,21.88,7.55,7.9,5.19
+LV,1.58,0,1.8,0.3
+LT,0.88,0.76,0.2,0.05
+LU,1.13,1.29,0,0
+NL,0.04,0,0,0.01
+NO,30.51,1.35,84.4,14
+PL,2.35,1.4,1.6,0.23
+PT,5.72,1.03,2.6,1.37
+RO,6.55,0.09,12.1,1.95
+RS,2.14,0.61,0,1.18
+SK,2.52,0.92,2.2,0.49
+SI,1.25,0.18,2.2,0.36
+ES,18.55,2.75,18.4,2.61
+SE,16.41,0.1,33.8,7.8
+CH,13.3,4.03,8.4,4.29

doc/configtables/enable.csv

@@ -2,11 +2,8 @@
 enable,str or bool,"{auto, true, false}","Switch to include (true) or exclude (false) the retrieve_* rules of snakemake into the workflow; 'auto' sets true|false based on availability of an internet connection to prevent issues with snakemake failing due to lack of internet connection."
 prepare_links_p_nom,bool,"{true, false}","Switch to retrieve current HVDC projects from `Wikipedia <https://en.wikipedia.org/wiki/List_of_HVDC_projects>`_"
 retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zenodo via the rule :mod:`retrieve_databundle` or whether to keep a custom databundle located in the corresponding folder."
-retrieve_sector_databundle,bool,"{true, false}","Switch to retrieve sector databundle from zenodo via the rule :mod:`retrieve_sector_databundle` or whether to keep a custom databundle located in the corresponding folder."
 retrieve_cost_data,bool,"{true, false}","Switch to retrieve technology cost data from `technology-data repository <https://github.com/PyPSA/technology-data>`_."
 build_cutout,bool,"{true, false}","Switch to enable the building of cutouts via the rule :mod:`build_cutout`."
 retrieve_cutout,bool,"{true, false}","Switch to enable the retrieval of cutouts from zenodo with :mod:`retrieve_cutout`."
-build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`."
-retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`."
 custom_busmap,bool,"{true, false}","Switch to enable the use of custom busmaps in rule :mod:`cluster_network`. If activated the rule looks for provided busmaps at ``data/custom_busmap_elec_s{simpl}_{clusters}.csv`` which should have the same format as ``resources/busmap_elec_s{simpl}_{clusters}.csv``, i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc``."
 drop_leap_day,bool,"{true, false}","Switch to drop February 29 from all time-dependent data in leap years"

doc/configtables/licenses.csv

@@ -5,10 +5,8 @@
 "naturalearth/*",,,,,http://www.naturalearthdata.com/about/terms-of-use/
 "NUTS_2013 _60M_SH/*","x","x",,"x",https://ec.europa.eu/eurostat/web/gisco/geodata/reference-data/administrative-units-statistical-units
 "cantons.csv","x",,"x",,https://en.wikipedia.org/wiki/Data_codes_for_Switzerland
-"eia_hydro_annual_generation.csv","x",,,,https://www.eia.gov/about/copyrights_reuse.php
-"GEBCO_2014_2D.nc","x",,,,https://www.gebco.net/data_and_products/gridded_bathymetry_data/documents/gebco_2014_historic.pdf
+"gebco/GEBCO_2014_2D.nc","x",,,,https://www.gebco.net/data_and_products/gridded_bathymetry_data/documents/gebco_2014_historic.pdf
 "hydro_capacities.csv","x",,,,
 "je-e-21.03.02.xls","x","x",,,https://www.bfs.admin.ch/bfs/en/home/fso/swiss-federal-statistical-office/terms-of-use.html
 "nama_10r_3 gdp.tsv.gz","x",,,"x",https://ec.europa.eu/eurostat/about/policies/copyright
 "nama_10r_3 popgdp.tsv.gz","x",,,"x",https://ec.europa.eu/eurostat/about/policies/copyright
-"time_series_60min _singleindex_filtered.csv","x",,,,https://data.open-power-system-data.org/time_series/2019-06-05/README.md

doc/configtables/load.csv

@@ -4,4 +4,4 @@ time_shift_for_large_gaps,string,string,"Periods which are used for copying time
 manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`."
 scaling_factor,--,float,"Global correction factor for the load time series."
 fixed_year,--,Year or False,"To specify a fixed year for the load time series that deviates from the snapshots' year"
-supplement_synthetic,bool,"{true, false}","Whether to supplement missing data for selected time period should be supplemented by synthetic data from https://zenodo.org/record/10820928."
+supplement_synthetic,bool,"{true, false}","Whether to supplement missing data for selected time period should be supplemented by synthetic data from https://zenodo.org/records/10820928."

doc/index.rst

@@ -74,7 +74,7 @@ greenhouse gas emitters except waste management, agriculture, forestry and land
 use. The diagram below gives an overview of the sectors and the links between
 them:
 
-.. image:: ../graphics/multisector_figure.png
+.. image:: img/multisector_figure.png
     :width: 70%
     :align: center
 

doc/licenses.rst

@@ -28,16 +28,13 @@ Electricity Systems Databundle
   More details are included in `the description of the
   data bundles on zenodo <https://zenodo.org/record/3517935#.XbGeXvzRZGo>`__.
 
-.. csv-table::
-   :header-rows: 1
-   :file: configtables/licenses.csv
-
 * BY: Attribute Source
 * NC: Non-Commercial Use Only
 * SA: Share Alike
 
-Sector-Coupled Systems Databundle
-=================================
+.. csv-table::
+   :header-rows: 1
+   :file: configtables/licenses.csv
 
 .. csv-table::
    :header-rows: 1

doc/preparation.rst

@@ -26,7 +26,6 @@ With these and the externally extracted ENTSO-E online map topology
 Then the process continues by calculating conventional power plant capacities, potentials, and per-unit availability time series for variable renewable energy carriers and hydro power plants with the following rules:
 
 - :mod:`build_powerplants` for today's thermal power plant capacities using `powerplantmatching <https://github.com/FRESNA/powerplantmatching>`__ allocating these to the closest substation for each powerplant,
-- :mod:`build_natura_raster` for rasterising NATURA2000 natural protection areas,
 - :mod:`build_ship_raster` for building shipping traffic density,
 - :mod:`build_renewable_profiles` for the hourly capacity factors and installation potentials constrained by land-use in each substation's Voronoi cell for PV, onshore and offshore wind, and
 - :mod:`build_hydro_profile` for the hourly per-unit hydro power availability time series.
@@ -47,14 +46,6 @@ Rule ``prepare_links_p_nom``
 
 .. automodule:: prepare_links_p_nom
 
-.. _natura:
-
-Rule ``build_natura_raster``
-===============================
-
-.. automodule:: build_natura_raster
-
-
 .. _base:
 
 Rule ``base_network``

doc/release_notes.rst

@@ -13,6 +13,28 @@ Upcoming Release
 
 * Group existing capacities to the earlier grouping_year for consistency with optimized capacities.
 
+* Update data bundle:
+
+  - Merge electricity-only and sector-coupled data bundles into `one bundle
+    <https://zenodo.org/records/10973944>`_. This means that the rule
+    ``retrieve_sector_databundle`` was removed.
+
+  - Include rasterised ``natura.tiff`` in data bundle and remove rule
+    ``retrieve_natura_raster``.
+
+  - Remove rule ``build_natura_raster`` as this rule is rarely run and increases
+    the data bundle size considerably.
+
+  - Remove outdated files from data bundle (e.g., Eurostat energy balances)
+
+  - Reduce spatial scope of GEBCO bathymetry data to Europe to save space.
+
+  - Remove the use of a separate data bundle for tutorials.
+
+  - Directly download `Hotmaps Industrial Database
+    <https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database/-/blob/master/data/Industrial_Database.csv>`__
+    from source and remove ``Industrial_Database.csv`` from data bundle.
+
 * bugfix: installed heating capacities were 5% lower than existing heating capacities
 
 * Include gas and oil fields and saline aquifers in estimation of CO2 sequestration potential.
@@ -1520,7 +1542,7 @@ This release is known to work with `PyPSA-Eur
 **Gas Transmission Network**
 
 * New rule ``retrieve_gas_infrastructure_data`` that downloads and extracts the
-  SciGRID_gas `IGGIELGN <https://zenodo.org/record/4767098>`__ dataset from
+  SciGRID_gas `IGGIELGN <https://zenodo.org/records/4767098>`__ dataset from
   zenodo. It includes data on the transmission routes, pipe diameters,
   capacities, pressure, and whether the pipeline is bidirectional and carries
   H-Gas or L-Gas.
@@ -1680,7 +1702,7 @@ This release is known to work with `PyPSA-Eur
   PyPSA network.
 
 * Updated `data bundle
-  <https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz>`__
+  <https://zenodo.org/records/5824485/files/pypsa-eur-sec-data-bundle.tar.gz>`__
   that includes the hydrogan salt cavern storage potentials.
 
 * Updated and extended documentation in
@@ -2040,7 +2062,7 @@ PyPSA-Eur-Sec codebase in Version 0.2.0 above.
 
 This model has `its own github repository
 <https://github.com/martavp/pypsa-eur-sec-30-path>`__ and is `archived
-on Zenodo <https://zenodo.org/record/4014807>`__.
+on Zenodo <https://zenodo.org/records/4014807>`__.
 
 
 
@@ -2056,7 +2078,7 @@ European countries with one node per country. It includes demand and
 supply for electricity, space and water heating in buildings, and land
 transport.
 
-It is `archived on Zenodo <https://zenodo.org/record/1146666>`__.
+It is `archived on Zenodo <https://zenodo.org/records/1146666>`__.
 
 
 Release Process

doc/retrieve.rst

@@ -53,32 +53,6 @@ The :ref:`tutorial` uses a smaller cutout than required for the full model (30 M
     For details see :mod:`build_cutout` and read the `atlite documentation <https://atlite.readthedocs.io>`__.
 
 
-Rule ``retrieve_natura_raster``
-================================
-
-.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.4706686.svg
-   :target: https://doi.org/10.5281/zenodo.4706686
-
-This rule, as a substitute for :mod:`build_natura_raster`, downloads an already rasterized version (`natura.tiff <https://zenodo.org/record/4706686/files/natura.tiff>`__) of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`__ natural protection areas to reduce computation times. The file is placed into the ``resources`` sub-directory.
-
-**Relevant Settings**
-
-.. code:: yaml
-
-    enable:
-        build_natura_raster:
-
-.. seealso::
-    Documentation of the configuration file ``config/config.yaml`` at
-    :ref:`toplevel_cf`
-
-**Outputs**
-
-- ``resources/natura.tiff``: Rasterized version of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`__ natural protection areas to reduce computation times.
-
-.. seealso::
-    For details see :mod:`build_natura_raster`.
-
 
 Rule ``retrieve_electricity_demand``
 ====================================
@@ -130,14 +104,3 @@ None.
 **Outputs**
 
 - ``data/shipdensity_global.zip``
-
-
-Rule ``retrieve_sector_databundle``
-====================================
-
-.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.5546516.svg
-   :target: https://doi.org/10.5281/zenodo.5546516
-
-In addition to the databundle required for electricity-only studies,
-another databundle is required for modelling sector-coupled systems.
-The size of this data bundle is around 640 MB.

doc/spatial_resolution.rst

@@ -15,11 +15,11 @@ The total number of nodes for Europe is set in the ``config/config.yaml`` file u
 
 Exemplary unsolved network clustered to 512 nodes:
 
-.. image:: ../graphics/elec_s_512.png
+.. image:: img/elec_s_512.png
 
 Exemplary unsolved network clustered to 37 nodes:
 
-.. image:: ../graphics/elec_s_37.png
+.. image:: img/elec_s_37.png
 
 The total number of nodes for Europe is set in the ``config/config.yaml`` file under `clusters <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L20>`__.  The number of nodes can vary between 37, the number of independent countries/synchronous areas, and several hundred. With 200-300 nodes, the model needs 100-150 GB RAM to solve with a commercial solver like Gurobi.
 Not all of the sectors are at the full nodal resolution, and some demand for some sectors is distributed to nodes using heuristics that need to be corrected. Some networks are copper-plated to reduce computational times.

doc/supply_demand.rst

@@ -18,7 +18,7 @@ management, carbon capture and usage/sequestration, and gas networks.
 
 The basic supply (left column) and demand (right column) options in the model are described in this figure:
 
-.. image:: ../graphics/multisector_figure.png
+.. image:: img/multisector_figure.png
 
 .. _Electricity supply and demand:
 
@@ -72,11 +72,11 @@ For every country, heat demand is split between low and high population density
 
 Cooling is electrified and is included in the electricity demand. Cooling demand is assumed to remain at current levels.  An example of regional distribution of the total heat demand for network 181 regions is depicted below.
 
-.. image:: ../graphics/demand-map-heat.png
+.. image:: img/demand-map-heat.png
 
 As below figure shows, the current total heat demand in Europe is similar to the total electricity demand but features much more pronounced seasonal variations. The current total building heating demand in Europe adds up to 3084 TWh/a of which 78% occurs in urban areas.
 
-.. image:: ../graphics/Heat_and_el_demand_timeseries.png
+.. image:: img/Heat_and_el_demand_timeseries.png
 
 In practice, in PyPSA-Eur-Sec, there are heat demand buses to which the corresponding heat demands are added.
 
@@ -269,7 +269,7 @@ The existing European gas transmission network is represented based on the SciGR
 
 The following figure shows the unclustered European gas transmission network based on the SciGRID Gas IGGIELGN dataset. Pipelines are color-coded by estimated capacities. Markers indicate entry-points, sites of fossil resource extraction, and LNG terminals.
 
-.. image:: ../graphics/gas_pipeline_figure.png
+.. image:: img/gas_pipeline_figure.png
 
 .. _Biomass supply:
 
@@ -374,7 +374,7 @@ Where process heat is required, our approach depends on the necessary temperatur
 
 The following figure shows the final consumption of energy and non-energy feedstocks in industry today in comparison to the scenario in 2050 assumed in `Neumann et al <https://arxiv.org/abs/2207.05816>`__.
 
-.. image:: ../graphics/fec_industry_today_tomorrow.png
+.. image:: img/fec_industry_today_tomorrow.png
 
 
 The following figure shows the process emissions in industry today (top bar) and in 2050 without
@@ -383,12 +383,12 @@ carbon capture (bottom bar) assumed in `Neumann et al <https://arxiv.org/abs/220
 
 
 
-.. image:: ../graphics/process-emissions.png
+.. image:: img/process-emissions.png
 
 
-Inside each country the industrial demand is then distributed using the `Hotmaps Industrial Database <https://zenodo.org/record/4687147#.YvOaxhxBy5c>`__, which is illustrated in the figure below. This open database includes georeferenced industrial sites of energy-intensive industry sectors in EU28, including cement, basic chemicals, glass, iron and steel, non-ferrous metals, non-metallic minerals, paper, and refineries subsectors. The use of this spatial dataset enables the calculation of regional and process-specific energy demands. This approach assumes that there will be no significant migration of energy-intensive industries.
+Inside each country the industrial demand is then distributed using the `Hotmaps Industrial Database <https://zenodo.org/records/4687147#.YvOaxhxBy5c>`__, which is illustrated in the figure below. This open database includes georeferenced industrial sites of energy-intensive industry sectors in EU28, including cement, basic chemicals, glass, iron and steel, non-ferrous metals, non-metallic minerals, paper, and refineries subsectors. The use of this spatial dataset enables the calculation of regional and process-specific energy demands. This approach assumes that there will be no significant migration of energy-intensive industries.
 
-.. image:: ../graphics/hotmaps.png
+.. image:: img/hotmaps.png
 
 
 .. _Iron and Steel: