Restyle Generalize retrieve databundle

Snakefile

@@ -56,7 +56,9 @@ datafiles = [
         "data/raw/gebco/GEBCO_2021_TID.nc",
         "data/raw/eez/eez_v11.gpkg",
         # "data/raw/landcover",  # set as an explicit directory in the rule
-        "data/raw/hydrobasins/hybas_lake_af_lev04_v1c.shp",
+        "data/raw/hydrobasins/hybas_world_lev04_v1c.shp",
+        "data/custom_powerplants.csv",
+        "data/hydro_capacities.csv",
         "data/costs.csv",
 ]
 
@@ -67,6 +69,7 @@ if config.get('tutorial')==True:
 
 if config['enable'].get('retrieve_databundle', True):
     rule retrieve_databundle_light:
+        input: "configs/bundle_config.yaml"
         output: #expand(directory('{file}') if isdir('{file}') else '{file}', file=datafiles)
             expand('{file}', file=datafiles),
             directory("data/raw/landcover")

config.default.yaml

@@ -196,7 +196,7 @@ renewable:
     cutout: africa-2013-era5
     resource:
       method: hydro
-      hydrobasins: data/raw/hydrobasins/hybas_lake_af_lev04_v1c.shp
+      hydrobasins: data/raw/hydrobasins/hybas_world_lev04_v1c.shp
       # flowspeed: 1.0
       # weight_with_height: false
       # show_progress: true

config.tutorial.yaml

@@ -226,7 +226,7 @@ renewable:
     cutout: africa-2013-era5-tutorial
     resource:
       method: hydro
-      hydrobasins: data/raw/hydrobasins/hybas_lake_af_lev04_v1c.shp
+      hydrobasins: data/raw/hydrobasins/hybas_world_lev04_v1c.shp
       # flowspeed: 1.0
       # weight_with_height: false
       # show_progress: true

configs/bundle_config.yaml

@@ -0,0 +1,113 @@
+# SPDX-FileCopyrightText: : 2021 The PyPSA-Africa Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+# File describing the data bundle configurations that can be downloaded
+# by using the retrieve_databundle fule
+
+# Each bundle shall have the following structure:
+
+# .. code:: yaml
+
+#   bundle_name:  # name of the bundle
+#     countries: [country code, region code or country list]  # list of countries represented in the databundle
+#     [tutorial: true/false]  # (optional, default false) whether the bundle is a tutorial or not
+#     category: common/resources/data/cutouts  # category of data contained in the bundle:
+#     destination: "."  # folder where to unzip the files with respect to the repository root (\"\" or \".\")
+#     urls:  # list of urls by source, e.g. zenodo or google
+#       zenodo: {zenodo url}
+#       google: {google url}
+
+
+# tutorial bundle specific for Nigeria and Benin only
+bundle_data_tutorial:
+  countries: [NG, BJ]
+  tutorial: true
+  category: data
+  destination: "data"
+  urls:
+    zenodo: https://sandbox.zenodo.org/record/1016522/files/tutorial_data.zip?download=1
+    google: https://drive.google.com/file/d/1Yd2iM892WFBRxh5NX22if6laRF1RxZKu/view?usp=sharing
+
+# tutorial bundle specific for Nigeria and Benin only
+bundle_resources_tutorial:
+  countries: [NG, BJ]
+  tutorial: true
+  category: resources
+  destination: "resources"
+  urls:
+    zenodo: https://sandbox.zenodo.org/record/1016001/files/tutorial_resources.zip?download=1
+    google: https://drive.google.com/file/d/1WWVyXHA2E7aGcg_OC1lN8Pb2tA8DgfU0/view?usp=sharing
+
+# tutorial bundle specific for Nigeria and Benin only
+bundle_cutouts_tutorial:
+  countries: [NG, BJ]
+  tutorial: true
+  category: cutouts
+  destination: "cutouts"
+  urls:
+    zenodo: https://sandbox.zenodo.org/record/1016015/files/tutorial_cutouts.zip?download=1
+    google: https://drive.google.com/file/d/1Lro4X92sVuD30C0B3sQ2jJU37uZ50DWi/view?usp=sharing
+
+# data bundle containing the data of the data folder common to all regions of the world
+bundle_data_world:
+  countries: [world]
+  category: common
+  destination: "data"
+  urls:
+    zenodo: https://sandbox.zenodo.org/record/1016540/files/common_data.zip?download=1
+    google: https://drive.google.com/file/d/1fo-YsyU2LKkVeg5Ea0JbcEx-zQ3kJKW7/view?usp=sharing
+
+# resources bundle containing the resources folder for africa only
+bundle_resources_africa:
+  countries: [africa]
+  category: resources
+  destination: "resources"
+  urls:
+    zenodo: https://sandbox.zenodo.org/record/1016042/files/resources_africa.zip?download=1
+    google: https://drive.google.com/file/d/1he31BBLtdemZt2dmBOwUCbP_jVuI3KS8/view?usp=sharing
+
+# data bundle of the data folder for the african continent
+bundle_landcover_africa:
+  countries: [africa]
+  category: landcover
+  destination: "data/raw/landcover"
+  urls:
+    zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_africa.zip?download=1
+    google: https://drive.google.com/file/d/1l6Xgr3PXyvgqWaXUbto9zvvMS_OgSjvG/view?usp=sharing
+
+# data bundle of the data folder for the african continent
+bundle_landcover_asiapacific:
+  countries: [asia, australia-oceania]
+  category: landcover
+  destination: "data/raw/landcover"
+  urls:
+    zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_asiapacific.zip?download=1
+    google: https://drive.google.com/file/d/1-bevGdoAltJ1useZAS9PqEMDJQLyBEn8/view?usp=sharing
+
+# data bundle of the data folder for the african continent
+bundle_landcover_latinamerica_caribbean:
+  countries: [latin_america, central_america]
+  category: landcover
+  destination: "data/raw/landcover"
+  urls:
+    zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_latinamerica_caribbean.zip?download=1
+    google: https://drive.google.com/file/d/16GiEqdUcX0juGHaAxk5q9XlgUl2T7eVf/view?usp=sharing
+
+# data bundle of the data folder for the african continent
+bundle_landcover_westasia:
+  countries: [WAS]
+  category: landcover
+  destination: "data/raw/landcover"
+  urls:
+    zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_westasia.zip?download=1
+    google: https://drive.google.com/file/d/1QDFn-USxTEToFrST8Iazj9Cz_mcXEwiP/view?usp=sharing
+
+# cutouts bundle of the cutouts folder for the african continent
+bundle_cutouts_africa:
+  countries: [africa]
+  category: cutouts
+  destination: "cutouts"
+  urls:
+    zenodo: https://sandbox.zenodo.org/record/1016042/files/cutouts_africa.zip?download=1
+    google: https://drive.google.com/file/d/1-Njs7BqG0YE5QwBHj0zgkdicb5IQvQCh/view

doc/release_notes.rst

@@ -21,7 +21,7 @@ Upcoming Release
 
 * Rule build_shape extended to the world
 
-* Replace google by zenodo data retrieval
+* Generalized version of Data retrieval with google and zenodo hosting platforms
 
 * Fix random state for kmean clustering, `adopted from PR <https://github.com/PyPSA/pypsa-eur/pull/313>`
 

envs/environment.yaml

@@ -35,7 +35,6 @@ dependencies:
   - networkx
   - scipy
   - shapely>=1.8a1
-  - progressbar2
   - pre-commit
   - pyomo
   - matplotlib

scripts/_helpers.py

@@ -268,12 +268,14 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False):
 def progress_retrieve(url, file):
     import urllib
 
-    from progressbar import ProgressBar
+    from tqdm import tqdm
 
-    pbar = ProgressBar(0, 100)
+    pbar = tqdm(total=100)
 
     def dlProgress(count, blockSize, totalSize):
-        pbar.update(int(count * blockSize * 100 / totalSize))
+        pbar.n = round(
+            count * blockSize * 100 / totalSize * 100) / 100  # round to 0.01
+        pbar.refresh()
 
     urllib.request.urlretrieve(url, file, reporthook=dlProgress)
 

scripts/build_renewable_profiles.py

@@ -252,13 +252,19 @@
 
     # filter plants for hydro
     if snakemake.wildcards.technology.startswith("hydro"):
+        country_shapes = gpd.read_file(paths.country_shapes)
         hydrobasins = gpd.read_file(resource["hydrobasins"])
-        merged_hydrobasin_shape = unary_union(hydrobasins.geometry)
+        hydrobasins = hydrobasins[[
+            any(country_shapes.geometry.intersects(geom))
+            for geom in hydrobasins.geometry
+        ]]  # exclude hydrobasins shapes that do not intersect the countries of interest
         resource["plants"] = regions.rename(columns={
             "x": "lon",
             "y": "lat"
         })[[
-            merged_hydrobasin_shape.intersects(p)
+            # select busbar whose location (p)
+            any(hydrobasins.geometry.intersects(p))
+            # belongs to at least one hydrobasin geometry
             for p in gpd.points_from_xy(regions.x, regions.y, crs=regions.crs)
         ]]  # TODO: filtering by presence of hydro generators should be the way to go