Merge pull request #1027 from PyPSA/reduced-data-bundle

update and reduce databundle size

.github/workflows/ci.yaml

@@ -19,7 +19,7 @@ on:
   - cron: "0 5 * * TUE"
 
 env:
-  DATA_CACHE_NUMBER: 1
+  DATA_CACHE_NUMBER: 2
 
 jobs:
   build:

.gitignore

@@ -37,18 +37,16 @@ dconf
 /data/links_p_nom.csv
 /data/*totals.csv
 /data/biomass*
-/data/bundle-sector/emobility/
-/data/bundle-sector/eea*
-/data/bundle-sector/jrc*
+/data/bundle/emobility/
+/data/bundle/eea*
+/data/bundle/jrc*
 /data/heating/
-/data/bundle-sector/eurostat*
+/data/bundle/eurostat*
 /data/odyssee/
 /data/transport_data.csv
-/data/bundle-sector/switzerland*
 /data/.nfs*
-/data/bundle-sector/Industrial_Database.csv
-/data/retro/tabula-calculator-calcsetbuilding.csv
-/data/bundle-sector/nuts*
+/data/retro/*
+/data/bundle/nuts*
 data/gas_network/scigrid-gas/
 data/costs_*.csv
 

config/config.default.yaml

@@ -69,12 +69,9 @@ enable:
   retrieve: auto
   prepare_links_p_nom: false
   retrieve_databundle: true
-  retrieve_sector_databundle: true
   retrieve_cost_data: true
   build_cutout: false
   retrieve_cutout: true
-  build_natura_raster: false
-  retrieve_natura_raster: true
   custom_busmap: false
   drop_leap_day: true
 

config/config.entsoe-all.yaml

@@ -38,6 +38,5 @@ lines:
 enable:
   retrieve: true
   retrieve_databundle: true
-  retrieve_sector_databundle: false
   retrieve_cost_data: true
   retrieve_cutout: true

data/ch_cantons.csv

@@ -0,0 +1,27 @@
+Canton,HASC,NUTS
+Aargau,CH.AG,CH033
+Appenzell Inner Rhodes,CH.AI,CH054
+Appenzell Outer Rhodes,CH.AR,CH053
+Basel-Landschaft,CH.BL,CH032
+Basel-Stadt,CH.BS,CH031
+Bern,CH.BE,CH021
+Fribourg,CH.FR,CH022
+Geneva,CH.GE,CH013
+Glarus,CH.GL,CH051
+Graubünden,CH.GR,CH056
+Jura,CH.JU,CH025
+Lucerne,CH.LU,CH061
+Neuchâtel,CH.NE,CH024
+Nidwalden,CH.NW,CH065
+Obwalden,CH.OW,CH064
+Sankt Gallen,CH.SG,CH055
+Schaffhausen,CH.SH,CH052
+Schwyz,CH.SZ,CH063
+Solothurn,CH.SO,CH023
+Thurgau,CH.TG,CH057
+Ticino,CH.TI,CH07
+Uri,CH.UR,CH062
+Valais,CH.VS,CH012
+Vaud,CH.VD,CH011
+Zug,CH.ZG,CH066
+Zurich,CH.ZH,CH04

data/hydro_capacities.csv

@@ -0,0 +1,31 @@
+Country,p_nom_discharge[GW],p_nom_store[GW],E_store[TWh],InflowHourlyAvg[GWh]
+AT,13.08,3.8,3.2,4.02
+BE,1.42,1.31,0,0.04
+BA,2.05,0.62,2.5,0.71
+BG,3.13,0.86,4,0.53
+HR,2,0.61,2.8,0.57
+CZ,2.21,0.68,1.5,0.24
+DK,0.01,0,0,0
+EE,0.01,0,0,0
+FI,3.2,0,5.5,1.59
+FR,25.37,6.99,9.8,7.82
+DE,11.26,6.8,0.3,1.93
+GB,4.43,2.74,0,0.46
+GR,3.24,0.7,2.3,0.26
+HU,0.06,0,0.1,0.02
+IE,0.53,0.29,0,0.08
+IT,21.88,7.55,7.9,5.19
+LV,1.58,0,1.8,0.3
+LT,0.88,0.76,0.2,0.05
+LU,1.13,1.29,0,0
+NL,0.04,0,0,0.01
+NO,30.51,1.35,84.4,14
+PL,2.35,1.4,1.6,0.23
+PT,5.72,1.03,2.6,1.37
+RO,6.55,0.09,12.1,1.95
+RS,2.14,0.61,0,1.18
+SK,2.52,0.92,2.2,0.49
+SI,1.25,0.18,2.2,0.36
+ES,18.55,2.75,18.4,2.61
+SE,16.41,0.1,33.8,7.8
+CH,13.3,4.03,8.4,4.29

doc/configtables/enable.csv

@@ -2,11 +2,8 @@
 enable,str or bool,"{auto, true, false}","Switch to include (true) or exclude (false) the retrieve_* rules of snakemake into the workflow; 'auto' sets true|false based on availability of an internet connection to prevent issues with snakemake failing due to lack of internet connection."
 prepare_links_p_nom,bool,"{true, false}","Switch to retrieve current HVDC projects from `Wikipedia <https://en.wikipedia.org/wiki/List_of_HVDC_projects>`_"
 retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zenodo via the rule :mod:`retrieve_databundle` or whether to keep a custom databundle located in the corresponding folder."
-retrieve_sector_databundle,bool,"{true, false}","Switch to retrieve sector databundle from zenodo via the rule :mod:`retrieve_sector_databundle` or whether to keep a custom databundle located in the corresponding folder."
 retrieve_cost_data,bool,"{true, false}","Switch to retrieve technology cost data from `technology-data repository <https://github.com/PyPSA/technology-data>`_."
 build_cutout,bool,"{true, false}","Switch to enable the building of cutouts via the rule :mod:`build_cutout`."
 retrieve_cutout,bool,"{true, false}","Switch to enable the retrieval of cutouts from zenodo with :mod:`retrieve_cutout`."
-build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`."
-retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`."
 custom_busmap,bool,"{true, false}","Switch to enable the use of custom busmaps in rule :mod:`cluster_network`. If activated the rule looks for provided busmaps at ``data/custom_busmap_elec_s{simpl}_{clusters}.csv`` which should have the same format as ``resources/busmap_elec_s{simpl}_{clusters}.csv``, i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc``."
 drop_leap_day,bool,"{true, false}","Switch to drop February 29 from all time-dependent data in leap years"

doc/configtables/licenses.csv

@@ -5,10 +5,8 @@
 "naturalearth/*",,,,,http://www.naturalearthdata.com/about/terms-of-use/
 "NUTS_2013 _60M_SH/*","x","x",,"x",https://ec.europa.eu/eurostat/web/gisco/geodata/reference-data/administrative-units-statistical-units
 "cantons.csv","x",,"x",,https://en.wikipedia.org/wiki/Data_codes_for_Switzerland
-"eia_hydro_annual_generation.csv","x",,,,https://www.eia.gov/about/copyrights_reuse.php
-"GEBCO_2014_2D.nc","x",,,,https://www.gebco.net/data_and_products/gridded_bathymetry_data/documents/gebco_2014_historic.pdf
+"gebco/GEBCO_2014_2D.nc","x",,,,https://www.gebco.net/data_and_products/gridded_bathymetry_data/documents/gebco_2014_historic.pdf
 "hydro_capacities.csv","x",,,,
 "je-e-21.03.02.xls","x","x",,,https://www.bfs.admin.ch/bfs/en/home/fso/swiss-federal-statistical-office/terms-of-use.html
 "nama_10r_3 gdp.tsv.gz","x",,,"x",https://ec.europa.eu/eurostat/about/policies/copyright
 "nama_10r_3 popgdp.tsv.gz","x",,,"x",https://ec.europa.eu/eurostat/about/policies/copyright
-"time_series_60min _singleindex_filtered.csv","x",,,,https://data.open-power-system-data.org/time_series/2019-06-05/README.md

doc/configtables/load.csv

@@ -4,4 +4,4 @@ time_shift_for_large_gaps,string,string,"Periods which are used for copying time
 manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`."
 scaling_factor,--,float,"Global correction factor for the load time series."
 fixed_year,--,Year or False,"To specify a fixed year for the load time series that deviates from the snapshots' year"
-supplement_synthetic,bool,"{true, false}","Whether to supplement missing data for selected time period should be supplemented by synthetic data from https://zenodo.org/record/10820928."
+supplement_synthetic,bool,"{true, false}","Whether to supplement missing data for selected time period should be supplemented by synthetic data from https://zenodo.org/records/10820928."

doc/licenses.rst

@@ -28,16 +28,13 @@ Electricity Systems Databundle
   More details are included in `the description of the
   data bundles on zenodo <https://zenodo.org/record/3517935#.XbGeXvzRZGo>`__.
 
-.. csv-table::
-   :header-rows: 1
-   :file: configtables/licenses.csv
-
 * BY: Attribute Source
 * NC: Non-Commercial Use Only
 * SA: Share Alike
 
-Sector-Coupled Systems Databundle
-=================================
+.. csv-table::
+   :header-rows: 1
+   :file: configtables/licenses.csv
 
 .. csv-table::
    :header-rows: 1

doc/preparation.rst

@@ -26,7 +26,6 @@ With these and the externally extracted ENTSO-E online map topology
 Then the process continues by calculating conventional power plant capacities, potentials, and per-unit availability time series for variable renewable energy carriers and hydro power plants with the following rules:
 
 - :mod:`build_powerplants` for today's thermal power plant capacities using `powerplantmatching <https://github.com/FRESNA/powerplantmatching>`__ allocating these to the closest substation for each powerplant,
-- :mod:`build_natura_raster` for rasterising NATURA2000 natural protection areas,
 - :mod:`build_ship_raster` for building shipping traffic density,
 - :mod:`build_renewable_profiles` for the hourly capacity factors and installation potentials constrained by land-use in each substation's Voronoi cell for PV, onshore and offshore wind, and
 - :mod:`build_hydro_profile` for the hourly per-unit hydro power availability time series.
@@ -47,14 +46,6 @@ Rule ``prepare_links_p_nom``
 
 .. automodule:: prepare_links_p_nom
 
-.. _natura:
-
-Rule ``build_natura_raster``
-===============================
-
-.. automodule:: build_natura_raster
-
-
 .. _base:
 
 Rule ``base_network``

doc/release_notes.rst

@@ -11,6 +11,28 @@ Upcoming Release
 ================
 * Group existing capacities to the earlier grouping_year for consistency with optimized capacities.
 
+* Update data bundle:
+
+  - Merge electricity-only and sector-coupled data bundles into `one bundle
+    <https://zenodo.org/records/10973944>`_. This means that the rule
+    ``retrieve_sector_databundle`` was removed.
+
+  - Include rasterised ``natura.tiff`` in data bundle and remove rule
+    ``retrieve_natura_raster``.
+
+  - Remove rule ``build_natura_raster`` as this rule is rarely run and increases
+    the data bundle size considerably.
+
+  - Remove outdated files from data bundle (e.g., Eurostat energy balances)
+
+  - Reduce spatial scope of GEBCO bathymetry data to Europe to save space.
+
+  - Remove the use of a separate data bundle for tutorials.
+
+  - Directly download `Hotmaps Industrial Database
+    <https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database/-/blob/master/data/Industrial_Database.csv>`__
+    from source and remove ``Industrial_Database.csv`` from data bundle.
+
 * bugfix: installed heating capacities were 5% lower than existing heating capacities
 
 * Include gas and oil fields and saline aquifers in estimation of CO2 sequestration potential.
@@ -1518,7 +1540,7 @@ This release is known to work with `PyPSA-Eur
 **Gas Transmission Network**
 
 * New rule ``retrieve_gas_infrastructure_data`` that downloads and extracts the
-  SciGRID_gas `IGGIELGN <https://zenodo.org/record/4767098>`__ dataset from
+  SciGRID_gas `IGGIELGN <https://zenodo.org/records/4767098>`__ dataset from
   zenodo. It includes data on the transmission routes, pipe diameters,
   capacities, pressure, and whether the pipeline is bidirectional and carries
   H-Gas or L-Gas.
@@ -1678,7 +1700,7 @@ This release is known to work with `PyPSA-Eur
   PyPSA network.
 
 * Updated `data bundle
-  <https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz>`__
+  <https://zenodo.org/records/5824485/files/pypsa-eur-sec-data-bundle.tar.gz>`__
   that includes the hydrogan salt cavern storage potentials.
 
 * Updated and extended documentation in
@@ -2038,7 +2060,7 @@ PyPSA-Eur-Sec codebase in Version 0.2.0 above.
 
 This model has `its own github repository
 <https://github.com/martavp/pypsa-eur-sec-30-path>`__ and is `archived
-on Zenodo <https://zenodo.org/record/4014807>`__.
+on Zenodo <https://zenodo.org/records/4014807>`__.
 
 
 
@@ -2054,7 +2076,7 @@ European countries with one node per country. It includes demand and
 supply for electricity, space and water heating in buildings, and land
 transport.
 
-It is `archived on Zenodo <https://zenodo.org/record/1146666>`__.
+It is `archived on Zenodo <https://zenodo.org/records/1146666>`__.
 
 
 Release Process

doc/retrieve.rst

@@ -53,32 +53,6 @@ The :ref:`tutorial` uses a smaller cutout than required for the full model (30 M
     For details see :mod:`build_cutout` and read the `atlite documentation <https://atlite.readthedocs.io>`__.
 
 
-Rule ``retrieve_natura_raster``
-================================
-
-.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.4706686.svg
-   :target: https://doi.org/10.5281/zenodo.4706686
-
-This rule, as a substitute for :mod:`build_natura_raster`, downloads an already rasterized version (`natura.tiff <https://zenodo.org/record/4706686/files/natura.tiff>`__) of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`__ natural protection areas to reduce computation times. The file is placed into the ``resources`` sub-directory.
-
-**Relevant Settings**
-
-.. code:: yaml
-
-    enable:
-        build_natura_raster:
-
-.. seealso::
-    Documentation of the configuration file ``config/config.yaml`` at
-    :ref:`toplevel_cf`
-
-**Outputs**
-
-- ``resources/natura.tiff``: Rasterized version of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`__ natural protection areas to reduce computation times.
-
-.. seealso::
-    For details see :mod:`build_natura_raster`.
-
 
 Rule ``retrieve_electricity_demand``
 ====================================
@@ -130,14 +104,3 @@ None.
 **Outputs**
 
 - ``data/shipdensity_global.zip``
-
-
-Rule ``retrieve_sector_databundle``
-====================================
-
-.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.5546516.svg
-   :target: https://doi.org/10.5281/zenodo.5546516
-
-In addition to the databundle required for electricity-only studies,
-another databundle is required for modelling sector-coupled systems.
-The size of this data bundle is around 640 MB.

doc/supply_demand.rst

@@ -386,7 +386,7 @@ carbon capture (bottom bar) assumed in `Neumann et al <https://arxiv.org/abs/220
 .. image:: img/process-emissions.png
 
 
-Inside each country the industrial demand is then distributed using the `Hotmaps Industrial Database <https://zenodo.org/record/4687147#.YvOaxhxBy5c>`__, which is illustrated in the figure below. This open database includes georeferenced industrial sites of energy-intensive industry sectors in EU28, including cement, basic chemicals, glass, iron and steel, non-ferrous metals, non-metallic minerals, paper, and refineries subsectors. The use of this spatial dataset enables the calculation of regional and process-specific energy demands. This approach assumes that there will be no significant migration of energy-intensive industries.
+Inside each country the industrial demand is then distributed using the `Hotmaps Industrial Database <https://zenodo.org/records/4687147#.YvOaxhxBy5c>`__, which is illustrated in the figure below. This open database includes georeferenced industrial sites of energy-intensive industry sectors in EU28, including cement, basic chemicals, glass, iron and steel, non-ferrous metals, non-metallic minerals, paper, and refineries subsectors. The use of this spatial dataset enables the calculation of regional and process-specific energy demands. This approach assumes that there will be no significant migration of energy-intensive industries.
 
 .. image:: img/hotmaps.png