diff --git a/data/catalogs/README.rst b/data/catalogs/README.rst index 5b97a8b5f..58007af48 100644 --- a/data/catalogs/README.rst +++ b/data/catalogs/README.rst @@ -80,24 +80,21 @@ structure per data set: - order the components of each data set alphabetically - for adding meta data use the following optional keys: -meta: - category: - notes: - paper_doi: - paper_ref: - source_author: (if different from paper_ref) - source_license: - source_url: - source_spatial_extent: - source_temporal_extent: - processing_script: - processing_notes: +category: +notes: +paper_doi: +paper_ref: +source_author: (if different from paper_ref) +source_license: +source_url: +source_version: +unit: updates ------- - create new branch on github - make changes and bump the version in the global meta section using `calendar versioning ` -- test your yml file (Can the added/changed data sources be read through HydroMT?), this can be done by running the test_data_catalog.py file located in this folder. Read the docstring for more information. +- test your yml file (Can the added/changed data sources be read through HydroMT?) - create pull request - add new version to hydromt\data\predefined_catalogs.yml diff --git a/data/catalogs/artifact_data.yml b/data/catalogs/artifact_data.yml index 3628c828a..e09b16148 100644 --- a/data/catalogs/artifact_data.yml +++ b/data/catalogs/artifact_data.yml @@ -1,7 +1,7 @@ meta: - version: v0.0.9 + version: v0.0.8 hydromt_version: '>=0.9, <1.0' - root: https://github.com/DirkEilander/hydromt-artifacts/releases/download/v0.0.9/data.tar.gz + root: https://github.com/DirkEilander/hydromt-artifacts/releases/download/v0.0.8/data.tar.gz chelsa: crs: 4326 @@ -13,9 +13,8 @@ chelsa: paper_ref: Karger et al. (2017) source_license: CC BY 4.0 source_url: http://chelsa-climate.org/downloads/ - version: 1.2 + source_version: 1.2 path: chelsa.tif - chirps_global: crs: 4326 data_type: RasterDataset @@ -26,11 +25,10 @@ chirps_global: paper_ref: Funk et al (2014) source_license: CC source_url: https://www.chc.ucsb.edu/data/chirps - version: 2.0 + source_version: v2.0 path: chirps_global.nc unit_add: time: 86400 - corine: data_type: RasterDataset driver: raster @@ -39,9 +37,8 @@ corine: source_author: European Environment Agency source_license: https://land.copernicus.eu/pan-european/corine-land-cover/clc2018?tab=metadata source_url: https://land.copernicus.eu/pan-european/corine-land-cover/clc2018 - version: 2020_20u1 + source_version: v.2020_20u1 path: corine.tif - dtu10mdt: crs: 4326 data_type: RasterDataset @@ -51,10 +48,9 @@ dtu10mdt: paper_doi: 10.1029/2008JC005179 paper_ref: Andersen and Knudsen (2009) source_url: https://www.space.dtu.dk/english/research/scientific_data_and_models/global_mean_dynamic_topography + source_version: 2010 unit: m+EGM2008 - version: 2010 path: dtu10mdt.tif - dtu10mdt_egm96: crs: 4326 data_type: RasterDataset @@ -64,10 +60,9 @@ dtu10mdt_egm96: paper_doi: 10.1029/2008JC005179 paper_ref: Andersen and Knudsen (2009) source_url: https://www.space.dtu.dk/english/research/scientific_data_and_models/global_mean_dynamic_topography + source_version: 2010 unit: m+EGM96 - version: 2010 path: dtu10mdt_egm96.tif - eobs: crs: 4326 data_type: RasterDataset @@ -78,11 +73,10 @@ eobs: paper_ref: Cornes et al (2018) source_license: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles source_url: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles - version: 22.0e + source_version: 22.0e path: eobs.nc unit_add: time: 86400 - eobs_orography: crs: 4326 data_type: RasterDataset @@ -93,9 +87,8 @@ eobs_orography: paper_ref: Cornes et al (2018) source_license: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles source_url: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles - version: 22.0e + source_version: 22.0e path: eobs_orography.nc - era5: crs: 4326 data_type: RasterDataset @@ -108,6 +101,7 @@ era5: paper_ref: Hersbach et al. (2019) source_license: https://cds.climate.copernicus.eu/cdsapp/#!/terms/licence-to-use-copernicus-products source_url: https://doi.org/10.24381/cds.bd0915c6 + source_version: ERA5 daily data on pressure levels path: era5.nc unit_add: temp: -273.15 @@ -119,7 +113,6 @@ era5: kout: 0.000277778 precip: 1000 press_msl: 0.01 - era5_hourly: crs: 4326 data_type: RasterDataset @@ -131,6 +124,7 @@ era5_hourly: paper_ref: Hersbach et al. (2019) source_license: https://cds.climate.copernicus.eu/cdsapp/#!/terms/licence-to-use-copernicus-products source_url: https://doi.org/10.24381/cds.bd0915c6 + source_version: ERA5 hourly data on pressure levels path: era5_hourly.nc unit_add: temp: -273.15 @@ -139,7 +133,6 @@ era5_hourly: kout: 0.000277778 precip: 1000 press_msl: 0.01 - era5_daily_zarr: crs: 4326 data_type: RasterDataset @@ -152,6 +145,7 @@ era5_daily_zarr: paper_ref: Hersbach et al. (2019) source_license: https://cds.climate.copernicus.eu/cdsapp/#!/terms/licence-to-use-copernicus-products source_url: https://doi.org/10.24381/cds.bd0915c6 + source_version: ERA5 daily data on pressure levels path: era5_daily_zarr.zarr rename: d2m: temp_dew @@ -161,7 +155,6 @@ era5_daily_zarr: temp_dew: -273.15 unit_mult: ssr: 0.000277778 - era5_hourly_zarr: crs: 4326 data_type: RasterDataset @@ -173,6 +166,7 @@ era5_hourly_zarr: paper_ref: Hersbach et al. (2019) source_license: https://cds.climate.copernicus.eu/cdsapp/#!/terms/licence-to-use-copernicus-products source_url: https://doi.org/10.24381/cds.bd0915c6 + source_version: ERA5 hourly data on pressure levels path: era5_hourly_zarr.zarr rename: d2m: temp_dew @@ -182,7 +176,6 @@ era5_hourly_zarr: temp_dew: -273.15 unit_mult: ssr: 0.000277778 - era5_orography: crs: 4326 data_type: RasterDataset @@ -194,10 +187,10 @@ era5_orography: paper_ref: Hersbach et al. (2019) source_license: https://cds.climate.copernicus.eu/cdsapp/#!/terms/licence-to-use-copernicus-products source_url: https://doi.org/10.24381/cds.bd0915c6 + source_version: ERA5 hourly data on pressure levels path: era5_orography.nc unit_mult: elevtn: 0.10197162129779283 - gadm_level1: crs: 4326 data_type: GeoDataFrame @@ -208,9 +201,8 @@ gadm_level1: source_author: gadm source_license: https://gadm.org/license.html source_url: https://gadm.org/download_world.html - version: 1.0 + source_version: 1.0 path: gadm_level1.gpkg - gadm_level2: crs: 4326 data_type: GeoDataFrame @@ -221,9 +213,8 @@ gadm_level2: source_author: gadm source_license: https://gadm.org/license.html source_url: https://gadm.org/download_world.html - version: 1.0 + source_version: 1.0 path: gadm_level2.gpkg - gadm_level3: crs: 4326 data_type: GeoDataFrame @@ -234,9 +225,8 @@ gadm_level3: source_author: gadm source_license: https://gadm.org/license.html source_url: https://gadm.org/download_world.html - version: 1.0 + source_version: 1.0 path: gadm_level3.gpkg - gcn250: data_type: RasterDataset driver: raster @@ -246,10 +236,9 @@ gcn250: paper_ref: Jaafar et al. (2019) source_license: CC BY 4.0 source_url: https://doi.org/10.6084/m9.figshare.7756202.v1 - version: 1 + source_version: v1 nodata: 255 path: gcn250/{variable}.tif - gdp_world: crs: 4326 data_type: GeoDataFrame @@ -260,9 +249,8 @@ gdp_world: source_info: data combined from World Bank (https://data.worldbank.org/indicator/NY.GDP.MKTP.CD) and CIA World Factbook (https://www.cia.gov/the-world-factbook/field/real-gdp-per-capita/country-comparison) source_license: CC BY-4.0 - version: 1.0 + source_version: 1.0 path: gdp_world.gpkg - gebco: crs: 4326 data_type: RasterDataset @@ -273,31 +261,22 @@ gebco: paper_ref: Weatherall et al (2020) source_license: https://www.gebco.net/data_and_products/gridded_bathymetry_data/#a1 source_url: https://www.bodc.ac.uk/data/open_download/gebco/gebco_2020/geotiff/ + source_version: 2020 unit: m+MSL - version: 2020 path: gebco.tif - -ghs_smod_2015: - crs: ESRI:54009 +ghs-smod_2015_v2: + crs: 54009 data_type: RasterDataset driver: raster meta: category: socio economic + paper_doi: 10.2905/42E8BE89-54FF-464E-BE7B-BF9E64DA5218 + paper_ref: Pesaresi et al (2019) source_author: JRC-ISPRA EC source_license: https://data.jrc.ec.europa.eu/licence/com_reuse - variants: - - version: R2016A_v1.0 - path: ghs_smod_2015.tif - meta: - paper_ref: Pesaresi and Freire (2016) - source_url: https://data.jrc.ec.europa.eu/dataset/jrc-ghsl-ghs_smod_pop_globe_r2016a - - version: R2019A_v2.0 - path: ghs-smod_2015_v2.tif - meta: - paper_doi: 10.2905/42E8BE89-54FF-464E-BE7B-BF9E64DA5218 - paper_ref: Pesaresi et al (2019) - source_url: https://data.jrc.ec.europa.eu/dataset/42e8be89-54ff-464e-be7b-bf9e64da5218 - + source_url: https://data.jrc.ec.europa.eu/dataset/42e8be89-54ff-464e-be7b-bf9e64da5218 + source_version: R2019A_v2.0 + path: ghs-smod_2015_v2.tif ghs_pop_2015: crs: 4326 data_type: RasterDataset @@ -309,11 +288,10 @@ ghs_pop_2015: source_author: JRC-ISPRA EC source_license: https://data.jrc.ec.europa.eu/licence/com_reuse source_url: https://data.jrc.ec.europa.eu/dataset/0c6b9751-a71f-4062-830b-43c9f432370f - version: R2019A_v1.0 + source_version: R2019A_v1.0 path: ghs_pop_2015.tif - ghs_pop_2015_54009: - crs: ESRI:54009 + crs: 54009 data_type: RasterDataset driver: raster meta: @@ -322,9 +300,19 @@ ghs_pop_2015_54009: paper_ref: Florczyk et al (2019) source_license: CC BY 4.0 source_url: https://ghsl.jrc.ec.europa.eu/download.php?ds=pop - version: R2019A_v1.0 path: ghs_pop_2015_54009.tif - +ghs_smod_2015: + crs: 54009 + data_type: RasterDataset + driver: raster + meta: + category: socio economic + paper_ref: Pesaresi and Freire (2016) + source_author: JRC-ISPRA EC + source_license: https://data.jrc.ec.europa.eu/licence/com_reuse + source_url: https://data.jrc.ec.europa.eu/dataset/jrc-ghsl-ghs_smod_pop_globe_r2016a + source_version: R2016A_v1.0 + path: ghs_smod_2015.tif globcover: crs: 4326 data_type: RasterDataset @@ -335,9 +323,7 @@ globcover: paper_ref: Arino et al (2012) source_license: CC-BY-3.0 source_url: http://due.esrin.esa.int/page_globcover.php - version: 2.3 path: globcover.tif - glw_buffaloes: crs: 4326 data_type: RasterDataset @@ -349,9 +335,8 @@ glw_buffaloes: source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: glw_buffaloes.tif - glw_cattle: crs: 4326 data_type: RasterDataset @@ -363,9 +348,8 @@ glw_cattle: source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: glw_cattle.tif - glw_chicken: crs: 4326 data_type: RasterDataset @@ -377,9 +361,8 @@ glw_chicken: source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: glw_chicken.tif - glw_ducks: crs: 4326 data_type: RasterDataset @@ -391,9 +374,8 @@ glw_ducks: source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: glw_ducks.tif - glw_goats: crs: 4326 data_type: RasterDataset @@ -405,9 +387,8 @@ glw_goats: source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: glw_goats.tif - glw_horses: crs: 4326 data_type: RasterDataset @@ -419,9 +400,8 @@ glw_horses: source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: glw_horses.tif - glw_pigs: crs: 4326 data_type: RasterDataset @@ -433,9 +413,8 @@ glw_pigs: source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: glw_pigs.tif - glw_sheep: crs: 4326 data_type: RasterDataset @@ -447,9 +426,8 @@ glw_sheep: source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: glw_sheep.tif - grdc: crs: 4326 data_type: GeoDataFrame @@ -460,7 +438,6 @@ grdc: source_license: https://www.bafg.de/GRDC/EN/02_srvcs/21_tmsrs/210_prtl/tou.html;jsessionid=A56D50D4A36D3D8707CBF00CBD71F106.live11291?nn=2862854 source_url: https://portal.grdc.bafg.de/applications/public.html?publicuser=PublicUser#dataDownload/StationCatalogue path: grdc.csv - grip_roads: crs: 4326 data_type: GeoDataFrame @@ -471,9 +448,8 @@ grip_roads: paper_ref: Meijer et al, 2018 source_license: CC0-1.0 source_url: https://www.globio.info/download-grip-dataset - version: 4 + source_version: v4 path: grip_roads.gpkg - grwl: data_type: GeoDataFrame driver: vector @@ -483,13 +459,12 @@ grwl: paper_ref: Allen and Pavelsky (2018) source_license: CC BY 4.0 source_url: https://doi.org/10.5281/zenodo.1297434 - version: 1.01 + source_version: 1.01 path: grwl.gpkg - grwl_mask: data_type: RasterDataset driver: raster_tindex - driver_kwargs: + kwargs: chunks: x: 3000 y: 3000 @@ -501,10 +476,9 @@ grwl_mask: paper_ref: Allen and Pavelsky (2018) source_license: CC BY 4.0 source_url: https://doi.org/10.5281/zenodo.1297434 - version: 1.01 + source_version: 1.01 nodata: 0 path: grwl_tindex.gpkg - gswo: data_type: RasterDataset driver: raster @@ -513,10 +487,9 @@ gswo: paper_doi: 10.1038/nature20584 paper_ref: Pekel et al. (2016) source_url: https://global-surface-water.appspot.com/download - version: 1.1 + source_version: v1_1_2019 nodata: 255 path: gswo.tif - gtsmv3_eu_era5: crs: 4326 data_type: GeoDataset @@ -527,10 +500,9 @@ gtsmv3_eu_era5: paper_ref: Copernicus Climate Change Service 2019 source_license: https://cds.climate.copernicus.eu/cdsapp/#!/terms/licence-to-use-copernicus-products source_url: https://cds.climate.copernicus.eu/cdsapp#!/dataset/10.24381/cds.8c59054f?tab=overview - version: 3.0 + source_version: GTSM v3.0 path: gtsmv3_eu_era5.nc - -guf: +guf_bld_2012: crs: 4326 data_type: RasterDataset driver: raster @@ -540,9 +512,7 @@ guf: paper_ref: Esch et al (2017) source_license: https://www.dlr.de/eoc/en/PortalData/60/Resources/dokumente/guf/DLR-GUF_LicenseAgreement-and-OrderForm.pdf source_url: http://www.dlr.de/guf - version: 2 path: guf_bld_2012.tif - hydro_lakes: crs: 4326 data_type: GeoDataFrame @@ -551,11 +521,10 @@ hydro_lakes: category: surface water source_author: Arjen Haag source_info: HydroLAKES.v10_extract - version: 1.0 + source_version: 1.0 path: hydro_lakes.gpkg unit_mult: Area_avg: 1000000.0 - hydro_reservoirs: crs: 4326 data_type: GeoDataFrame @@ -564,7 +533,7 @@ hydro_reservoirs: category: surface water source_author: Alessia Matano source_info: GRanD.v1.1_HydroLAKES.v10_JRC.2016 - version: 1.0 + source_version: 1.0 nodata: -99 path: hydro_reservoirs.gpkg unit_mult: @@ -573,7 +542,6 @@ hydro_reservoirs: Capacity_min: 1000000.0 Capacity_norm: 1000000.0 Vol_avg: 1000000.0 - koppen_geiger: crs: 4326 data_type: RasterDataset @@ -583,10 +551,9 @@ koppen_geiger: paper_doi: 10.1127/0941-2948/2006/0130 paper_ref: Kottek et al. (2006) source_url: http://koeppen-geiger.vu-wien.ac.at/present.htm - version: 2017 + source_version: 2017 nodata: 0 path: koppen_geiger.tif - mdt_cnes_cls18: crs: 4326 data_type: RasterDataset @@ -596,10 +563,9 @@ mdt_cnes_cls18: paper_doi: 10.5194/os-17-789-2021 paper_ref: Mulet et al (2021) source_url: https://www.aviso.altimetry.fr/en/data/products/auxiliary-products/mdt.html + source_version: 18 unit: m+GOCO05S - version: 18 path: mdt_cnes_cls18.tif - merit_hydro: crs: 4326 data_type: RasterDataset @@ -610,9 +576,8 @@ merit_hydro: paper_ref: Yamazaki et al. (2019) source_license: CC-BY-NC 4.0 or ODbL 1.0 source_url: http://hydro.iis.u-tokyo.ac.jp/~yamadai/MERIT_Hydro - version: 1.0 + source_version: 1.0 path: merit_hydro/{variable}.tif - merit_hydro_1k: crs: 4326 data_type: RasterDataset @@ -623,9 +588,8 @@ merit_hydro_1k: paper_ref: Eilander et al. (in review) source_license: CC-BY-NC 4.0 source_url: https://doi.org/10.5281/zenodo.4138776 - version: 0.1 + source_version: 0.1 path: merit_hydro_1k/{variable}.tif - merit_hydro_index: crs: 4326 data_type: GeoDataFrame @@ -635,9 +599,7 @@ merit_hydro_index: paper_doi: 10.5194/hess-2020-582 paper_ref: Eilander et al. (in review) source_license: CC-BY-NC 4.0 - version: 1.0 path: merit_hydro_index.gpkg - modis_lai: crs: 4326 data_type: RasterDataset @@ -650,11 +612,10 @@ modis_lai: paper_ref: Myneni et al (2015) source_license: https://lpdaac.usgs.gov/data/data-citation-and-policies/ source_url: https://lpdaac.usgs.gov/products/mcd15a3hv006/ - version: MCD15A3H V006 + source_version: MCD15A3H V006 path: modis_lai.nc unit_mult: LAI: 0.1 - osm_coastlines: crs: 4326 data_type: GeoDataFrame @@ -665,9 +626,8 @@ osm_coastlines: source_info: OpenStreetMap coastlines water polygons, last updated 2020-01-09T05:29 source_license: ODbL source_url: https://osmdata.openstreetmap.de/data/coastlines.html - version: 1.0 + source_version: 1.0 path: osm_coastlines.gpkg - osm_landareas: crs: 4326 data_type: GeoDataFrame @@ -678,9 +638,8 @@ osm_landareas: source_info: OpenStreetMap coastlines land polygons, last updated 2020-01-09T05:29 source_license: ODbL source_url: https://osmdata.openstreetmap.de/data/coastlines.html - version: 1.0 + source_version: 1.0 path: osm_landareas.gpkg - rgi: crs: 4326 data_type: GeoDataFrame @@ -692,10 +651,9 @@ rgi: source_info: Randolph Glacier Inventory source_license: CC BY 4.0 source_url: https://cds.climate.copernicus.eu/cdsapp#!/dataset/insitu-glaciers-extent?tab=overview - version: 6.0 + source_version: 6.0 path: rgi.gpkg - -hydro_rivers_lin2019: +rivers_lin2019_v1: data_type: GeoDataFrame driver: vector meta: @@ -704,9 +662,8 @@ hydro_rivers_lin2019: paper_ref: Lin et al. (2019) source_license: CC-BY-NC 4.0 source_url: https://zenodo.org/record/3552776#.YVbOrppByUk - version: 1 + source_version: 1 path: rivers_lin2019_v1.gpkg - simard: crs: 4326 data_type: RasterDataset @@ -716,9 +673,7 @@ simard: paper_doi: 10.1029/2011JG001708 paper_ref: Simard et al (2011) source_url: https://webmap.ornl.gov/ogc/dataset.jsp?ds_id=10023 - version: 2011 path: simard.tif - soilgrids: crs: 4326 data_type: RasterDataset @@ -736,7 +691,7 @@ soilgrids: paper_ref: Hengl et al. (2017) source_license: ODbL source_url: https://www.isric.org/explore/soilgrids/faq-soilgrids-2017 - version: 2017 + source_version: 2017 path: soilgrids/{variable}.tif unit_mult: bd_sl1: 0.001 @@ -760,7 +715,6 @@ soilgrids: ph_sl5: 0.1 ph_sl6: 0.1 ph_sl7: 0.1 - vito: crs: 4326 data_type: RasterDataset @@ -770,9 +724,8 @@ vito: paper_doi: 10.5281/zenodo.3939038 paper_ref: Buchhorn et al (2020) source_url: https://land.copernicus.eu/global/products/lc - version: 2.0.2 + source_version: v2.0.2 path: vito.tif - wb_countries: crs: 4326 data_type: GeoDataFrame @@ -783,9 +736,7 @@ wb_countries: source_license: CC-BY 4.0 source_url: https://datacatalog.worldbank.org/dataset/world-bank-official-boundaries timestamp: February 2020 - version: 2020319 path: wb_countries.gpkg - worldclim: crs: 4326 data_type: RasterDataset @@ -795,6 +746,5 @@ worldclim: paper_doi: 10.1002/joc.5086 paper_ref: Fick and Hijmans (2017) source_url: https://www.worldclim.org/data/worldclim21.html - version: 2 - nodata: -999.0 + source_version: 2 path: worldclim.nc diff --git a/data/catalogs/changelog.md b/data/catalogs/changelog.md deleted file mode 100644 index c294edcd6..000000000 --- a/data/catalogs/changelog.md +++ /dev/null @@ -1,139 +0,0 @@ -# Change log predefined datasets - -## deltares_data - -### version 2024.2 - -#### added -- version argument to data source where applicable -- variants argument for data sources that are of the same dataset but different versions -- processing_script / processing_notes arguments to data sources that have been (pre-) processed -- temporal extent of datasets that have a temporal dimension. -- spatial extents to datasets - -#### changed -- removed source_version from data source meta -- kwargs to driver_kwargs -- updated source_url if url was not working anymore -- sorted datasets by alphabetical order -- Removed version from dataset names and prefixed 'hydro' where necessary, see table below for mapping of old and new names. - -| Old name | New name | -|--------------------------------|---------------------------| -| basin_atlas_level12_v10 | hydro_basin_atlas_level12 | -| eobs_v.. | eobs | -| eobs_orography_v.. | eobs_orography | -| lake_atlas_pol_v10 | hydro_lake_atlas_pol | -| river_atlas_v10 | hydro_river_atlas | -| ghs_pop_2015_54009_v2019a | ghs_pop_2015 | -| ghs_smod_2015_54009_v2019a | ghs_smod_2015 | -| glofas_era5_v31 | glofas_era5 | -| guf_bld_2012 | guf_bld_2012 | -| rivers_lin2019_v1 | hydro_rivers_lin2019 | -| SM2RAIN_ASCAT_monthly_025_v1.4 | SM2RAIN_ASCAT_monthly_025 | -| SM2RAIN_ASCAT_monthly_05_v1.4 | SM2RAIN_ASCAT_monthly_05 | -| vito | vito_2015 | -| vito_2015_v2.0.2 | vito_2015 | -| vito_2016_v3.0.1 | vito_2016 | -| vito_2017_v3.0.1 | vito_2017 | -| vito_2018_v3.0.1 | vito_2018 | -| vito_2019_v3.0.1 | vito_2019 | - -- Some datasets have multiple versions, for these datasets the default can be changed if you do not supply a version in your config file. See the table below for which dataset the default version has changed. - -| Dataset name | Default version | Previous default version | -|----------------|-----------------|--------------------------| -| eobs | 25.0e | 22.0e | -| eobs_orography | 25.0e | 22.0e | - - -### version 2024.1.30 - -#### added -- HydroMT version to catalog -- GRDC dataset - - -### version: 2023.12 - -#### changed -- updated GADM dataset and converted the GeoPackage layers to FlatGeoBuf files -- removed gtsm_codec_reanalysis dataset - -#### added -- Added waterdemand pcr_globwb dataset -- Added GADM 4.1 as FlatGeoBuff files to deltares_data catalog (#686) - - -### version: 2023.2 - -#### changed -- convert GeoPackage files to FlatGeoBuf for cloud compatibility -- fix ERA5 nc files to read from archive of combined yearly and monthly files - -#### added -- Additional variables to era5 daily and hourly with name and unit conventions - - temp_dew: dewpoint temperature (degree C) - - wind10_u: 10m wind U-component (m s-1) - - wind10_v: 10m wind V-component (m s-1) - - ssr: surface net solar radiation (W m-2) - - tcc: total cloud cover (-) - - -### version: 2022.7 - -#### added - -- README with conventions regarding data (download, storage, .yml) -- change log file of deltares_data.yml -- new data sets - - basin_atlas_level12_v10 - - river_atlas_v10 - - lake_atlas_pol_v10 - - eobs_v24.0e - - eobs_v25.0e - - eobs_orography_v24.0e - - eobs_orography_v25.0e - - SM2RAIN_ASCAT_monthly_025_v1.4 - - SM2RAIN_ASCAT_monthly_05_v1.4 - -#### changed -- Apply convention specified in the README - - check reasonable alphabetical order in data sets and components - - implement right versioning convention _v where possible - - apply consistent meta information - -#### fixed -- enable versioning of yml.files - -## aws_data - -### version 0.0.9 - -#### changed -- updated source naming to match deltares_data version 2024.2 - -## cmip6_data - -### version: 2024.1.30 - -#### added -- hydromt_version to data catalog meta - - -### version: 2023.2 - -#### added -- CMIP6 data from Google Cloud Storage. Only models and scenarios for which regular grids are available are listed - -## aws_data - -### version: 2024.1.30 - -#### added -- HydroMT version to catalog - -### version: 2023.2 - -#### added -- ESA Worldcover v100 2020. diff --git a/data/catalogs/changelog.rst b/data/catalogs/changelog.rst new file mode 100644 index 000000000..0748c3a89 --- /dev/null +++ b/data/catalogs/changelog.rst @@ -0,0 +1,109 @@ +============================== +Change log predefined datasets +============================== + +deltares_data +============= + +version: 2024.1.30 +--------------- + +added +^^^^^ +- HydroMT version to catalog +- GRDC dataset + + +version: 2023.12 +----------------- + +changed +^^^^^^^ +- Updated GADM dataset and converted the GeoPackage layers to FlatGeoBuf files + +added +^^^^^ +- Added waterdemand pcr_globwb dataset +- Added GADM 4.1 as FlatGeoBuff files to deltares_data catalog (#686) + + +version: 2023.2 +---------------- + +changed +^^^^^^^ +- convert GeoPackage files to FlatGeoBuf for cloud compatibility +- fix ERA5 nc files to read from archive of combined yearly and monthly files + +added +^^^^^ +- Additional variables to era5 daily and hourly with name and unit conventions + - temp_dew: dewpoint temperature (degree C) + - wind10_u: 10m wind U-component (m s-1) + - wind10_v: 10m wind V-component (m s-1) + - ssr: surface net solar radiation (W m-2) + - tcc: total cloud cover (-) + + +version: 2022.7 +--------------- + +added +^^^^^ +- README with conventions regarding data (download, storage, .yml) +- change log file of deltares_data.yml +- new data sets + - basin_atlas_level12_v10 + - river_atlas_v10 + - lake_atlas_pol_v10 + - eobs_v24.0e + - eobs_v25.0e + - eobs_orography_v24.0e + - eobs_orography_v25.0e + - SM2RAIN_ASCAT_monthly_025_v1.4 + - SM2RAIN_ASCAT_monthly_05_v1.4 + +changed +^^^^^^^ +- Apply convention specified in the README + - check reasonable alphabetical order in data sets and components + - implement right versioning convention _v where possible + - apply consistent meta information + +fixed +^^^^^ +- enable versioning of yml.files + +cmip6_data +========== + +version: 2024.1.30 +--------------- + +added +^^^^^ +- HydroMT version to catalog + +version: 2023.2 +--------------- + +added +^^^^^ +- CMIP6 data from Google Cloud Storage. Only models and scenarios for which regular grids are available are listed + +aws_data +======== + +version: 2024.1.30 +--------------- + +added +^^^^^ +- HydroMT version to catalog + +version: 2023.2 +--------------- + +added +^^^^^ +- ESA Worldcover v100 2020. diff --git a/data/catalogs/deltares_data.yml b/data/catalogs/deltares_data.yml index f017217dd..5dbeb7c49 100644 --- a/data/catalogs/deltares_data.yml +++ b/data/catalogs/deltares_data.yml @@ -1,15 +1,15 @@ --- meta: root: p:/wflow_global/hydromt - version: 2024.2 + version: 2024.1.30 hydromt_version: '>=0.9, <1.0' -hydro_basin_atlas_level12: +basin_atlas_level12_v10: crs: 4326 data_type: GeoDataFrame driver: vector - driver_kwargs: + kwargs: layer: BasinATLAS_v10_lev12 meta: category: hydrography @@ -18,19 +18,50 @@ hydro_basin_atlas_level12: paper_ref: Linke et al. (2019) source_license: CC BY 4.0 source_url: https://www.hydrosheds.org/hydroatlas - source_spatial_extent: - West: -180.0 - South: -55.988 - East: 180.001 - North: 83.626 - version: 10 + source_version: 10 path: hydrography/hydro_atlas/basin_atlas_v10.gpkg +river_atlas_v10: + crs: 4326 + data_type: GeoDataFrame + driver: vector + meta: + category: hydrography + notes: renaming and units might require some revision + paper_doi: 10.1038/s41597-019-0300-6 + paper_ref: Linke et al. (2019) + source_license: CC BY 4.0 + source_url: https://www.hydrosheds.org/hydroatlas + source_version: 10 + path: hydrography/hydro_atlas/river_atlas_v10.gpkg + rename: + dis_m3_pyr: Dis_avg + +lake_atlas_pol_v10: + crs: 4326 + data_type: GeoDataFrame + driver: vector + kwargs: + layer: LakeATLAS_v10_pol + meta: + category: hydrography + notes: renaming and units might require some revision + paper_doi: 10.1038/s41597-022-01425-z + paper_ref: Lehner et al. (2022) + source_license: CC BY 4.0 + source_url: https://www.hydrosheds.org/hydroatlas + source_version: 10 + path: hydrography/hydro_atlas/lake_atlas_v10.gpkg + rename: + dis_m3_pyr: Dis_avg + chelsa: + alias: chelsa_v1.2 +chelsa_v1.2: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -40,19 +71,16 @@ chelsa: paper_ref: Karger et al. (2017) source_license: CC BY 4.0 source_url: http://chelsa-climate.org/downloads/ - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 84.0 - version: 1.2 + source_version: 1.2 path: meteo/chelsa_clim_v1.2/CHELSA_bio10_12.tif chirps: + alias: chirps_africa_daily_v2.0 +chirps_africa_daily_v2.0: crs: 4326 data_type: RasterDataset driver: netcdf - driver_kwargs: + kwargs: chunks: lat: 160 lon: 150 @@ -66,15 +94,7 @@ chirps: paper_ref: Funk et al (2015) source_license: CC source_url: https://www.chc.ucsb.edu/data/chirps - source_temporal_extent: - start: '1981-01-02' - end: '2022-04-01' - source_spatial_extent: - West: -20.0 - South: -40.0 - East: 55.0 - North: 40.0 - version: 2.0 + source_version: v2.0 path: meteo/chirps_africa_daily_v2.0/CHIRPS_rainfall_{year}.nc rename: precipitation: precip @@ -82,10 +102,12 @@ chirps: time: 86400 chirps_global: + alias: chirps_global_daily_v2.0 +chirps_global_daily_v2.0: crs: 4326 data_type: RasterDataset driver: netcdf - driver_kwargs: + kwargs: chunks: latitude: 112 longitude: 400 @@ -99,15 +121,7 @@ chirps_global: paper_ref: Funk et al (2014) source_license: CC source_url: https://www.chc.ucsb.edu/data/chirps - source_temporal_extent: - start: '1981-01-02' - end: '2023-02-01' - source_spatial_extent: - West: -180.0 - South: -50.0 - East: 180.0 - North: 50.0 - version: 2.0 + source_version: v2.0 path: meteo/chirps_global_daily_v2.0/chirps-v2.0.{year}.days_p05.nc rename: precipitation: precip @@ -115,74 +129,67 @@ chirps_global: time: 86400 copdem30: + alias: copdem30_v2021.1 +copdem30_v2021.1: data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 meta: category: topography source_license: https://spacedata.copernicus.eu/documents/20126/0/CSCDA_ESA_Mission-specific+Annex+%281%29.pdf/83b44c0a-244a-7ba3-b00c-b578a34e88a7?t=1604070311399 - source_url: https://spacedata.copernicus.eu/web/guest/collections/copernicus-digital-elevation-model?p_l_back_url=%2Fweb%2Fguest%2Fsearch%3Fq%3Ddemhttps://spacedata.copernicus.eu/web/cscda/dataset-details?articleId=394198#D4 - source_spatial_extent: - West: -180.001 - South: -90.0 - East: 180.0 - North: 84.0 - version: 2021.1 + source_url: https://spacedata.copernicus.eu/web/cscda/dataset-details?articleId=394198#D4 + source_version: '2021_1' + nodata: -9999 # copdem has no actual nodata, this is to suppress a missing nodata warning path: topography/copdem/copdem.vrt - nodata: -9999 rename: copdem: elevtn copdem30_mask: + alias: copdem30_mask_v2021.1 +copdem30_mask_v2021.1: data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 meta: category: topography source_license: https://spacedata.copernicus.eu/documents/20126/0/CSCDA_ESA_Mission-specific+Annex+%281%29.pdf/83b44c0a-244a-7ba3-b00c-b578a34e88a7?t=1604070311399 - source_url: https://spacedata.copernicus.eu/web/guest/collections/copernicus-digital-elevation-model?p_l_back_url=%2Fweb%2Fguest%2Fsearch%3Fq%3Ddemhttps://spacedata.copernicus.eu/web/cscda/dataset-details?articleId=394198#D4 - source_spatial_extent: - West: -180.001 - South: -90.0 - East: 180.0 - North: 84.0 - version: 2021.1 + source_url: https://spacedata.copernicus.eu/web/cscda/dataset-details?articleId=394198#D4 + source_version: '2021_1' + nodata: -1 # copdem has no actual nodata, this is to suppress a missing nodata warning path: topography/copdem/copdem_mask.vrt - nodata: -1 rename: copdem_mask: mask copdem30_masked: + alias: copdem30_masked_v2021.1 +copdem30_masked_v2021.1: data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 meta: category: topography source_license: https://spacedata.copernicus.eu/documents/20126/0/CSCDA_ESA_Mission-specific+Annex+%281%29.pdf/83b44c0a-244a-7ba3-b00c-b578a34e88a7?t=1604070311399 - source_url: https://spacedata.copernicus.eu/web/guest/collections/copernicus-digital-elevation-model?p_l_back_url=%2Fweb%2Fguest%2Fsearch%3Fq%3Ddemhttps://spacedata.copernicus.eu/web/cscda/dataset-details?articleId=394198#D4 - source_spatial_extent: - West: -180.001 - South: -90.0 - East: 180.0 - North: 84.0 - version: 2021.1 + source_url: https://spacedata.copernicus.eu/web/cscda/dataset-details?articleId=394198#D4 + source_version: '2021_1' path: topography/copdem/copdem_masked.vrt rename: copdem_masked: elevtn corine: + alias: corine_2018_v2020_u1 +corine_2018_v2020_u1: data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 6000 y: 6000 @@ -191,20 +198,14 @@ corine: source_author: European Environment Agency source_license: https://land.copernicus.eu/pan-european/corine-land-cover/clc2018?tab=metadata source_url: https://land.copernicus.eu/pan-european/corine-land-cover/clc2018 - source_spatial_extent: - West: 900000.0 - South: 900000.0 - East: 7400000.0 - North: 5500000.0 - provider: local - version: 2020_20u1 + source_version: v.2020_20u1 path: landuse/corine/CLC2018_CLC2018_V2018_20.tif dtu10mdt: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -213,20 +214,15 @@ dtu10mdt: paper_doi: 10.1029/2008JC005179 paper_ref: Andersen and Knudsen (2009) source_url: https://www.space.dtu.dk/english/research/scientific_data_and_models/global_mean_dynamic_topography + source_version: 2010 unit: m+EGM2008 - source_spatial_extent: - West: -179.992 - South: -89.992 - East: 179.992 - North: 89.992 - version: 2010 path: topography/dtu10mdt/DTU10MDT_1min_float32.tif dtu10mdt_egm96: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -235,22 +231,17 @@ dtu10mdt_egm96: paper_doi: 10.1029/2008JC005179 paper_ref: Andersen and Knudsen (2009) source_url: https://www.space.dtu.dk/english/research/scientific_data_and_models/global_mean_dynamic_topography - processing_notes: /topography/dtu10mdt/READ_ME.txt - processing_script: /topography/dtu10mdt/DTU10MDT_egm96_1min.ipynb + source_version: 2010 unit: m+EGM96 - source_spatial_extent: - West: -179.992 - South: -89.985 - East: 179.995 - North: 89.992 - version: 2010 path: topography/dtu10mdt/DTU10MDT_1min_egm96.tif eobs: + alias: eobs_v22.0e +eobs_v22.0e: crs: 4326 data_type: RasterDataset driver: netcdf - driver_kwargs: + kwargs: chunks: latitude: 100 longitude: 100 @@ -264,43 +255,8 @@ eobs: paper_ref: Cornes et al (2018) source_license: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles source_url: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles - source_temporal_extent: - start: '1950-01-02' - end: '2022-01-01' - source_spatial_extent: - West: -25.0 - South: 25.0 - East: 45.5 - North: 71.5 - variants: - - version: 22.0e - path: meteo/eobs/*/{variable}_ens_mean_0.1deg_reg_v22.0e.nc - meta: - source_temporal_extent: - start: '1950-01-02' - end: '2020-07-01' - - version: 23.1e - path: meteo/eobs/*/{variable}_ens_mean_0.1deg_reg_v23.1e.nc - meta: - source_temporal_extent: - start: '1950-01-02' - end: '2021-01-01' - - version: 24.0e - path: meteo/eobs/*/{variable}_ens_mean_0.1deg_reg_v24.0e.nc - meta: - processing_script: meteo/eobs/scripts/v24.0e_fix_qq_dims_and_match_extent.py - processing_notes: correct lat/lon dimension for variable qq to match the grid of the other variables - source_temporal_extent: - start: '1950-01-02' - end: '2021-07-01' - - version: 25.0e - path: meteo/eobs/*/{variable}_ens_mean_0.1deg_reg_v25.0e.nc - meta: - processing_script: v25.0e_fix_qq_dims_and_match_extent.py - processing_notes: correct lat/lon dimension for variable qq to match the grid of the other variables - source_temporal_extent: - start: '1950-01-02' - end: '2022-01-01' + source_version: 22.0e + path: meteo/eobs/*/{variable}_ens_mean_0.1deg_reg_v22.0e.nc rename: pp: press_msl qq: kin @@ -309,11 +265,116 @@ eobs: unit_add: time: 86400 +eobs_v23.1e: + crs: 4326 + data_type: RasterDataset + driver: netcdf + kwargs: + chunks: + latitude: 100 + longitude: 100 + time: 100 + combine: by_coords + parallel: true + preprocess: round_latlon + meta: + category: meteo + paper_doi: 10.1029/2017JD028200 + paper_ref: Cornes et al (2018) + source_license: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles + source_url: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles + source_version: 23.1e + path: meteo/eobs/*/{variable}_ens_mean_0.1deg_reg_v23.1e.nc + rename: + pp: press_msl + qq: kin + rr: precip + tg: temp + unit_add: + time: 86400 + +eobs_v24.0e: + crs: 4326 + data_type: RasterDataset + driver: netcdf + kwargs: + chunks: + latitude: 100 + longitude: 100 + time: 100 + combine: by_coords + parallel: true + preprocess: round_latlon + meta: + category: meteo + paper_doi: 10.1029/2017JD028200 + paper_ref: Cornes et al (2018) + source_license: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles + source_url: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles + source_version: 24.0e + path: meteo/eobs/*/{variable}_ens_mean_0.1deg_reg_v24.0e.nc + rename: + pp: press_msl + qq: kin + rr: precip + tg: temp + unit_add: + time: 86400 + +eobs_v25.0e: + crs: 4326 + data_type: RasterDataset + driver: netcdf + kwargs: + chunks: + latitude: 100 + longitude: 100 + time: 100 + combine: by_coords + parallel: true + preprocess: round_latlon + meta: + category: meteo + paper_doi: 10.1029/2017JD028200 + paper_ref: Cornes et al (2018) + source_license: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles + source_url: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles + source_version: 25.0e + path: meteo/eobs/*/{variable}_ens_mean_0.1deg_reg_v25.0e.nc + rename: + pp: press_msl + qq: kin + rr: precip + tg: temp + unit_add: + time: 86400 + +eobs_orography_v20.0e: + crs: 4326 + data_type: RasterDataset + driver: netcdf + kwargs: + chunks: + latitude: 100 + longitude: 100 + meta: + category: meteo + paper_doi: 10.1029/2017JD028200 + paper_ref: Cornes et al (2018) + source_license: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles + source_url: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles + source_version: 20.0e + path: meteo/eobs/elev_ens_0.1deg_reg_v20.0e.nc + rename: + elevation: elevtn + eobs_orography: + alias: eobs_orography_v22.0e +eobs_orography_v22.0e: crs: 4326 data_type: RasterDataset driver: netcdf - driver_kwargs: + kwargs: chunks: latitude: 100 longitude: 100 @@ -323,30 +384,75 @@ eobs_orography: paper_ref: Cornes et al (2018) source_license: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles source_url: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles - source_spatial_extent: - West: -25.0 - South: 25.0 - East: 45.5 - North: 71.5 - variants: - - version: 20.0e - path: meteo/eobs/elev_ens_0.1deg_reg_v20.0e.nc - - version: 22.0e - path: meteo/eobs/elev_ens_0.1deg_reg_v22.0e.nc - - version: 23.1e - path: meteo/eobs/elev_ens_0.1deg_reg_v23.1e.nc - - version: 24.0e - path: meteo/eobs/elev_ens_0.1deg_reg_v24.0e.nc - - version: 25.0e - path: meteo/eobs/elev_ens_0.1deg_reg_v25.0e.nc + source_version: 22.0e + path: meteo/eobs/elev_ens_0.1deg_reg_v22.0e.nc + rename: + elevation: elevtn + +eobs_orography_v23.1e: + crs: 4326 + data_type: RasterDataset + driver: netcdf + kwargs: + chunks: + latitude: 100 + longitude: 100 + meta: + category: meteo + paper_doi: 10.1029/2017JD028200 + paper_ref: Cornes et al (2018) + source_license: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles + source_url: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles + source_version: 23.1e + path: meteo/eobs/elev_ens_0.1deg_reg_v23.1e.nc + rename: + elevation: elevtn + +eobs_orography_v24.0e: + crs: 4326 + data_type: RasterDataset + driver: netcdf + kwargs: + chunks: + latitude: 100 + longitude: 100 + meta: + category: meteo + paper_doi: 10.1029/2017JD028200 + paper_ref: Cornes et al (2018) + source_license: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles + source_url: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles + source_version: 24.0e + path: meteo/eobs/elev_ens_0.1deg_reg_v24.0e.nc + rename: + elevation: elevtn + +eobs_orography_v25.0e: + crs: 4326 + data_type: RasterDataset + driver: netcdf + kwargs: + chunks: + latitude: 100 + longitude: 100 + meta: + category: meteo + paper_doi: 10.1029/2017JD028200 + paper_ref: Cornes et al (2018) + source_license: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles + source_url: https://surfobs.climate.copernicus.eu/dataaccess/access_eobs.php#datafiles + source_version: 25.0e + path: meteo/eobs/elev_ens_0.1deg_reg_v25.0e.nc rename: elevation: elevtn era5: + alias: era5_daily +era5_daily: crs: 4326 data_type: RasterDataset driver: netcdf - driver_kwargs: + kwargs: chunks: latitude: 250 longitude: 240 @@ -356,22 +462,13 @@ era5: parallel: true meta: category: meteo - notes: Extracted from Copernicus Climate Data Store; resampled by Deltares to - daily frequency using right label for time. + notes: Extracted from Copernicus Climate Data Store; resampled by Deltares to daily frequency paper_doi: 10.1002/qj.3803 paper_ref: Hersbach et al. (2019) - processing_script: https://github.com/Deltares/hydromt/blob/main/data/src/era5_download_resample_convert.py source_license: https://cds.climate.copernicus.eu/cdsapp/#!/terms/licence-to-use-copernicus-products source_url: https://doi.org/10.24381/cds.bd0915c6 - source_temporal_extent: - start: '1950-01-02' - end: '2023-11-30' - source_spatial_extent: - West: -0.125 - South: -90.125 - East: 359.875 - North: 90.125 - path: meteo/era5_daily/nc_merged/era5_{year}_daily.nc + source_version: ERA5 daily data on pressure levels + path: meteo/era5_daily/nc_merged/era5_{year}*_daily.nc rename: d2m: temp_dew msl: press_msl @@ -395,35 +492,19 @@ era5: press_msl: 0.01 era5_daily_zarr: + alias: era5_daily crs: 4326 data_type: RasterDataset driver: zarr - driver_kwargs: + kwargs: chunks: auto - meta: - category: meteo - notes: Extracted from Copernicus Climate Data Store; resampled by Deltares to - daily frequency - paper_doi: 10.1002/qj.3803 - paper_ref: Hersbach et al. (2019) - processing_script: https://github.com/Deltares/hydromt/blob/main/data/src/era5_download_resample_convert.py - source_license: https://cds.climate.copernicus.eu/cdsapp/#!/terms/licence-to-use-copernicus-products - source_url: https://doi.org/10.24381/cds.bd0915c6 - source_temporal_extent: - start: '1950-01-02' - end: '2023-02-01' - source_spatial_extent: - West: -0.125 - South: -90.125 - East: 359.875 - North: 90.125 path: meteo/era5_daily.zarr era5_hourly: crs: 4326 data_type: RasterDataset driver: netcdf - driver_kwargs: + kwargs: chunks: latitude: 125 longitude: 120 @@ -438,15 +519,8 @@ era5_hourly: paper_ref: Hersbach et al. (2019) source_license: https://cds.climate.copernicus.eu/cdsapp/#!/terms/licence-to-use-copernicus-products source_url: https://doi.org/10.24381/cds.adbb2d47 - source_temporal_extent: - start: '1950-01-01' - end: '2023-03-01' - source_spatial_extent: - West: -0.125 - South: -90.125 - East: 359.875 - North: 90.125 - path: meteo/era5/{variable}/era5_{variable}_{year}_hourly.nc + source_version: ERA5 hourly data on single levels from 1940 to present + path: meteo/era5/{variable}/era5_{variable}_{year}*_hourly.nc rename: d2m: temp_dew msl: press_msl @@ -467,34 +541,19 @@ era5_hourly: press_msl: 0.01 era5_hourly_zarr: + alias: era5_hourly crs: 4326 data_type: RasterDataset driver: zarr - driver_kwargs: + kwargs: chunks: auto - meta: - category: meteo - notes: Extracted from Copernicus Climate Data Store - paper_doi: 10.1002/qj.3803 - paper_ref: Hersbach et al. (2019) - source_license: https://cds.climate.copernicus.eu/cdsapp/#!/terms/licence-to-use-copernicus-products - source_url: https://doi.org/10.24381/cds.adbb2d47 - processing_script: meteo/era5_zarr/scripts/ - source_temporal_extent: - start: '1950-01-01' - end: '2023-02-01' - source_spatial_extent: - West: -0.125 - South: -90.125 - East: 359.875 - North: 90.125 path: meteo/era5.zarr era5_ocean: crs: 4326 data_type: RasterDataset driver: netcdf - driver_kwargs: + kwargs: chunks: latitude: 60 longitude: 60 @@ -509,21 +568,13 @@ era5_ocean: paper_ref: Hersbach et al. (2019) source_license: https://cds.climate.copernicus.eu/cdsapp/#!/terms/licence-to-use-copernicus-products source_url: https://doi.org/10.24381/cds.bd0915c6 - source_temporal_extent: - start: '1979-01-01' - end: '2022-04-08' - source_spatial_extent: - West: -0.25 - South: -90.25 - East: 359.75 - North: 90.25 path: ocean/era5/{variable}/era5_{variable}_{year}_hourly.nc era5_orography: crs: 4326 data_type: RasterDataset driver: netcdf - driver_kwargs: + kwargs: chunks: latitude: 120 longitude: 125 @@ -534,14 +585,6 @@ era5_orography: paper_ref: Hersbach et al. (2019) source_license: https://cds.climate.copernicus.eu/cdsapp/#!/terms/licence-to-use-copernicus-products source_url: https://apps.ecmwf.int/codes/grib/param-db?id=129 - source_temporal_extent: - start: '2018-01-01' - end: '2018-01-01' - source_spatial_extent: - West: -0.125 - South: -90.125 - East: 359.875 - North: 90.125 path: meteo/era5/meta/era5_orography_2018.nc rename: z: elevtn @@ -549,31 +592,27 @@ era5_orography: elevtn: 0.10197162129779283 esa_worldcover: + alias: esa_worldcover_2020_v100 +esa_worldcover_2020_v100: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 36000 y: 36000 meta: category: landuse - processing_notes: prepare vrt file with gdalbuildvrt source_license: CC BY 4.0 source_url: https://doi.org/10.5281/zenodo.5571936 - source_spatial_extent: - West: -180.0 - South: -60.0 - East: 180.0 - North: 84.0 - version: 100 + source_version: v100 path: landuse/esa_worldcover/esa-worldcover.vrt fabdem: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -583,16 +622,14 @@ fabdem: paper_ref: Hawker et al. (2022) source_license: CC BY-NC-SA 4.0 source_url: https://data.bris.ac.uk/data/dataset/25wfy0f9ukoge2gs7a5mqpq2j7 - source_spatial_extent: - West: -180.0 - South: -60.0 - East: 180.0 - North: 80.0 - version: 1.0 + source_version: 1.0 path: topography/fabdem/fabdem.vrt rename: fabdem: elevtn +gadm: + alias: gadm_level0 + gadm_level0: crs: 4326 data_type: GeoDataFrame @@ -603,13 +640,8 @@ gadm_level0: source_author: gadm source_license: https://gadm.org/license.html source_url: https://gadm.org/download_world.html + source_version: 4.1 processing_script: geography/gadm/version4_1/run_dissolve_level_0.sh - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 83.658 - version: 4.1 path: geography/gadm/version4_1/gadm_level0.fgb gadm_level1: @@ -622,13 +654,8 @@ gadm_level1: source_author: gadm source_license: https://gadm.org/license.html source_url: https://gadm.org/download_world.html + source_version: 4.1 processing_script: geography/gadm/version4_1/run_dissolve_level_1.sh - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 83.658 - version: 4.1 path: geography/gadm/version4_1/gadm_level1.fgb gadm_level2: @@ -641,13 +668,8 @@ gadm_level2: source_author: gadm source_license: https://gadm.org/license.html source_url: https://gadm.org/download_world.html + source_version: 4.1 processing_script: geography/gadm/version4_1/run_dissolve_level_2.sh - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 83.658 - version: 4.1 path: geography/gadm/version4_1/gadm_level2.fgb gadm_level3: @@ -660,19 +682,16 @@ gadm_level3: source_author: gadm source_license: https://gadm.org/license.html source_url: https://gadm.org/download_world.html + source_version: 4.1 processing_script: geography/gadm/version4_1/run_dissolve_level_3.sh - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 83.658 - version: 4.1 path: geography/gadm/version4_1/gadm_level3.fgb gcn250: + alias: gcn250_v1 +gcn250_v1: data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -682,13 +701,8 @@ gcn250: paper_ref: Jaafar et al. (2019) source_license: CC BY 4.0 source_url: https://doi.org/10.6084/m9.figshare.7756202.v1 - source_spatial_extent: - West: -180.0 - South: -56.0 - East: 180.0 - North: 84.0 + source_version: v1 nodata: 255 - version: 1 path: landuse/gcn250/GCN250_*.tif rename: GCN250_ARCI: cn_dry @@ -696,21 +710,17 @@ gcn250: GCN250_ARCIII: cn_wet gdp_world: + alias: gdp_world_v1 +gdp_world_v1: crs: 4326 data_type: GeoDataFrame driver: vector meta: category: socio-economic - processing_notes: data combined by Deltares (Wilfred Altena) from World Bank (https://data.worldbank.org/indicator/NY.GDP.MKTP.CD) - and CIA World Factbook (https://www.cia.gov/the-world-factbook/field/real-gdp-per-capita/country-comparison) + notes: data combined from World Bank (https://data.worldbank.org/indicator/NY.GDP.MKTP.CD) and CIA World Factbook (https://www.cia.gov/the-world-factbook/field/real-gdp-per-capita/country-comparison) source_author: Wilfred Altena source_license: CC BY-4.0 - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 83.665 - version: 1.0 + source_version: 1.0 path: socio_economic/gdp_world/World_countries_GDPpcPPP.fgb rename: GDP: gdp @@ -718,10 +728,12 @@ gdp_world: gdp: 0.001 gebco: + alias: gebco_v2020 +gebco_v2020: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -730,24 +742,20 @@ gebco: paper_doi: 10.5285/a29c5465-b138-234d-e053-6c86abc040b9 paper_ref: Weatherall et al (2020) source_license: https://www.gebco.net/data_and_products/gridded_bathymetry_data/#a1 - source_url: https://www.bodc.ac.uk/data/hosted_data_systems/gebco_gridded_bathymetry_data/ + source_url: https://www.bodc.ac.uk/data/open_download/gebco/gebco_2020/geotiff/ + source_version: 2020 unit: m+MSL - processing_script: bathymetry/gebco/gebco_tiles.ipynb - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 - version: 2020 path: bathymetry/gebco/gebco.vrt rename: gebco: elevtn -ghs_pop_2015: +ghs_pop: + alias: ghs_pop_2015_54009_v2019a +ghs_pop_2015_4326_v2019a: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -758,19 +766,14 @@ ghs_pop_2015: source_author: JRC-ISPRA EC source_license: https://data.jrc.ec.europa.eu/licence/com_reuse source_url: https://data.jrc.ec.europa.eu/dataset/0c6b9751-a71f-4062-830b-43c9f432370f - source_spatial_extent: - West: -180.0 - South: -59.485 - East: 180.0 - North: 83.628 - version: R2019A_v1.0 + source_version: R2019A_v1.0 path: socio_economic/ghs/GHS_POP_E2015_GLOBE_R2019A_4326_9ss_V1_0.tif -ghs_pop_2015_54009: - crs: ESRI:54009 +ghs_pop_2015_54009_v2019a: + crs: "ESRI:54009" data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 1000 y: 1000 @@ -780,49 +783,52 @@ ghs_pop_2015_54009: paper_ref: Florczyk et al (2019) source_license: CC BY 4.0 source_url: https://ghsl.jrc.ec.europa.eu/download.php?ds=pop - source_spatial_extent: - West: -18041000.0 - South: -9000000.0 - East: 18041000.0 - North: 9000000.0 - version: R2019A_v1.0 path: socio_economic/ghs/GHS_POP_E2015_GLOBE_R2019A_54009_250_V1_0.tif -ghs_smod_2015: - crs: ESRI:54009 +ghs_smod: + alias: ghs_smod_2015_54009_v2019a +ghs_smod_2015_54009_v2016a: + crs: "ESRI:54009" data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 meta: category: socio-economic + paper_ref: Pesaresi and Freire (2016) source_author: JRC-ISPRA EC source_license: https://data.jrc.ec.europa.eu/licence/com_reuse - source_spatial_extent: - West: -18041000.0 - South: -9000000.0 - East: 18041000.0 - North: 9000000.0 - variants: - - version: R2016A_v1.0 - path: socio_economic/ghs/GHS_SMOD_POP2015_GLOBE_R2016A_54009_1k_v1_0.tif - meta: - paper_ref: Pesaresi and Freire (2016) - source_url: https://data.jrc.ec.europa.eu/dataset/jrc-ghsl-ghs_smod_pop_globe_r2016a - - version: R2019A_v2.0 - path: socio_economic/ghs/GHS_SMOD_POP2015_GLOBE_R2019A_54009_1K_V2_0.tif - meta: - paper_doi: 10.2905/42E8BE89-54FF-464E-BE7B-BF9E64DA5218 - paper_ref: Pesaresi et al (2019) - source_url: https://data.jrc.ec.europa.eu/dataset/42e8be89-54ff-464e-be7b-bf9e64da5218 + source_url: https://data.jrc.ec.europa.eu/dataset/jrc-ghsl-ghs_smod_pop_globe_r2016a + source_version: R2016A_v1.0 + path: socio_economic/ghs/GHS_SMOD_POP2015_GLOBE_R2016A_54009_1k_v1_0.tif + +ghs_smod_2015_54009_v2019a: + crs: "ESRI:54009" + data_type: RasterDataset + driver: raster + kwargs: + chunks: + x: 3600 + y: 3600 + meta: + category: socio-economic + paper_doi: 10.2905/42E8BE89-54FF-464E-BE7B-BF9E64DA5218 + paper_ref: Pesaresi et al (2019) + source_author: JRC-ISPRA EC + source_license: https://data.jrc.ec.europa.eu/licence/com_reuse + source_url: https://data.jrc.ec.europa.eu/dataset/42e8be89-54ff-464e-be7b-bf9e64da5218 + source_version: R2019A_v2.0 + path: socio_economic/ghs/GHS_SMOD_POP2015_GLOBE_R2019A_54009_1K_V2_0.tif globcover: + alias: globcover_2009_v2.3 +globcover_2009_v2.3: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -832,19 +838,14 @@ globcover: paper_ref: Arino et al (2012) source_license: CC-BY-3.0 source_url: http://due.esrin.esa.int/page_globcover.php - source_spatial_extent: - West: -180.001 - South: -64.999 - East: 179.999 - North: 90.001 - version: 2.3 + source_version: v2.3 path: landuse/globcover/GLOBCOVER_L4_200901_200912_V2.3.tif -glofas_era5: +glofas_era5_v31: crs: 4326 data_type: RasterDataset driver: netcdf - driver_kwargs: + kwargs: chunks: time: 180 latitude: 150 @@ -859,75 +860,51 @@ glofas_era5: paper_ref: Harrigan et al. (2020) source_license: https://cds.climate.copernicus.eu/api/v2/terms/static/cems-floods.pdf source_url: https://doi.org/10.24381/cds.a4fdd6b9 - processing_script: hydro/glofas_era5/glofas_cds_get.py - source_temporal_extent: - start: '1980-01-01' - end: '2021-05-31' - source_spatial_extent: - West: -180.0 - South: -60.0 - East: 180.0 - North: 90.0 - version: 31 + source_version: v31 path: hydro/glofas_era5/glofas_v31_{year}.nc rename: dis24: discharge glofas_uparea: - data_type: RasterDataset + alias: glofas_era5_v31 driver: raster - driver_kwargs: + kwargs: chunks: x: 150 y: 360 - meta: - category: hydro - notes: Extracted from Copernicus Climate Data Store - paper_doi: 10.5194/essd-12-2043-2020 - paper_ref: Harrigan et al. (2020) - source_license: https://cds.climate.copernicus.eu/api/v2/terms/static/cems-floods.pdf - source_url: https://doi.org/10.24381/cds.a4fdd6b9 - source_spatial_extent: - West: -180.0 - South: -60.0 - East: 180.0 - North: 90.0 - version: 31 path: hydro/glofas_era5/glofas_v31_uparea.tif rename: glofas_v31_uparea: uparea unit_mult: - uparea: 1.0e-06 + uparea: 0.000001 glw_buffaloes: + alias: glw_buffaloes_v3 +glw_buffaloes_v3: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 meta: category: socio-economic - notes: last downloaded 2020-06-11 paper_doi: 10.7910/DVN/5U8MWI paper_ref: Gilbert at al (2018) source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: socio_economic/glw/5_Bf_2010_Da.tif glw_cattle: + alias: glw_cattle_v3 +glw_cattle_v3: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -938,19 +915,16 @@ glw_cattle: source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: socio_economic/glw/5_Ct_2010_Da.tif glw_chicken: + alias: glw_chicken_v3 +glw_chicken_v3: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -961,19 +935,16 @@ glw_chicken: source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: socio_economic/glw/5_Ch_2010_Da.tif glw_ducks: + alias: glw_ducks_v3 +glw_ducks_v3: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -984,19 +955,16 @@ glw_ducks: source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: socio_economic/glw/5_Dk_2010_Da.tif glw_goats: + alias: glw_goats_v3 +glw_goats_v3: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -1007,19 +975,16 @@ glw_goats: source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: socio_economic/glw/5_Gt_2010_Da.tif glw_horses: + alias: glw_horses_v3 +glw_horses_v3: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -1030,19 +995,16 @@ glw_horses: source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: socio_economic/glw/5_Ho_2010_Da.tif glw_pigs: + alias: glw_pigs_v3 +glw_pigs_v3: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -1053,19 +1015,16 @@ glw_pigs: source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: socio_economic/glw/5_Pg_2010_Da.tif glw_sheeps: + alias: glw_sheeps_v3 +glw_sheeps_v3: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -1076,19 +1035,15 @@ glw_sheeps: source_author: glw (Gridded Livestock of World 3 Dataverse) source_license: CC 4.0 source_url: https://dataverse.harvard.edu/dataverse/glw_3 - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 - version: 3 + source_version: GLW 3, last downloaded 2020-06-11 path: socio_economic/glw/5_Sh_2010_Da.tif + grdc: crs: 4326 data_type: GeoDataFrame driver: csv - driver_kwargs: + kwargs: index_col: grdc_no meta: category: observed data @@ -1100,6 +1055,8 @@ grdc: area: uparea grip_roads: + alias: grip_roads_v4 +grip_roads_v4: crs: 4326 data_type: GeoDataFrame driver: vector @@ -1109,19 +1066,15 @@ grip_roads: paper_ref: Meijer et al, 2018 paper_doi: 10.1088/1748-9326/aabd42 source_license: CC0-1.0 - processing_script: infrastructure/grip/merge.ipynb - source_spatial_extent: - West: -179.999 - South: -55.055 - East: 180.0 - North: 73.836 - version: 4 + source_version: v4 path: infrastructure/grip/GRIP4_world.fgb rename: GP_RTP: road_type GP_RCY: country_code grwl: + alias: grwl_v1.01 +grwl_v1.01: data_type: GeoDataFrame driver: vector meta: @@ -1130,22 +1083,19 @@ grwl: paper_ref: Allen and Pavelsky (2018) source_license: CC BY 4.0 source_url: https://doi.org/10.5281/zenodo.1297434 - source_spatial_extent: - West: -180.0 - South: -54.31 - East: 180.0 - North: 82.311 - version: 1.01 + source_version: 1.01 path: hydrography/grwl/GRWL_vector_V01.01/grwl.fgb grwl_mask: + alias: grwl_mask_v1.01 +grwl_mask_v1.01: data_type: RasterDataset driver: raster_tindex - driver_kwargs: + kwargs: chunks: x: 3000 y: 3000 - mosaic_driver_kwargs: + mosaic_kwargs: method: nearest tileindex: location meta: @@ -1154,14 +1104,16 @@ grwl_mask: paper_ref: Allen and Pavelsky (2018) source_license: CC BY 4.0 source_url: https://doi.org/10.5281/zenodo.1297434 + source_version: 1.01 nodata: 0 - version: 1.01 path: hydrography/grwl/tindex.gpkg gswo: + alias: gswo_v2019 +gswo_v2019: data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 4000 y: 4000 @@ -1170,20 +1122,40 @@ gswo: paper_doi: 10.1038/nature20584 paper_ref: Pekel et al. (2016) source_url: https://global-surface-water.appspot.com/download - source_spatial_extent: - West: -180.0 - South: -60.0 - East: 180.0 - North: 80.0 + source_version: v1_1_2019 nodata: 255 - version: 1.1 path: hydrography/gswo/occur.vrt +gtsm_codec_reanalysis_{freq}_v1: + crs: 4326 + data_type: GeoDataset + driver: netcdf + kwargs: + chunks: + stations: 10 + time: -1 + meta: + category: ocean + paper_doi: 10.3389/fmars.2020.00263 + paper_ref: Muis at al (2020) + source_license: https://cds.climate.copernicus.eu/api/v2/terms/static/licence-to-use-copernicus-products.pdf + source_url: https://doi.org/10.24381/cds.8c59054f + source_version: v1 + path: p:/11205028-c3s_435/01_data/01_Timeseries/timeseries2/{variable}/reanalysis_{variable}_{freq}_{year}_{month:02d}_v1.nc + placeholders: + freq: [10min, hourly, dailymax] + rename: + station_x_coordinate: lon + station_y_coordinate: lat + stations: index + guf_bld_2012: + alias: guf_bld_2012_v02 +guf_bld_2012_v02: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 1000 y: 1000 @@ -1193,12 +1165,6 @@ guf_bld_2012: paper_ref: Esch et al (2017) source_license: https://www.dlr.de/eoc/en/PortalData/60/Resources/dokumente/guf/DLR-GUF_LicenseAgreement-and-OrderForm.pdf source_url: http://www.dlr.de/guf - source_spatial_extent: - West: -180.0 - South: -65.0 - East: 180.0 - North: 85.0 - version: 2 path: infrastructure/guf/GUF04_DLR_v02.vrt hydro_lakes: @@ -1207,14 +1173,9 @@ hydro_lakes: driver: vector meta: category: surface water - processing_notes: extract of hydrolakes to only include natural lakes and exclude reservoirs and control lakes (ie filter on Lake_type = 1) - source_author: HydroSHEDS - source_spatial_extent: - West: -180.0 - South: -55.865 - East: 180.0 - North: 83.576 - version: 1.0 + notes: HydroLAKES.v10_extract + source_author: Arjen Haag + source_version: 1.0 path: hydrography/lakes/lake-db.fgb rename: Depth_avg: Depth_avg @@ -1235,13 +1196,8 @@ hydro_reservoirs: category: surface water notes: GRanD.v1.1_HydroLAKES.v10_JRC.2016 source_author: Alessia Matano - source_spatial_extent: - West: -153.059 - South: -45.881 - East: 176.825 - North: 70.396 + source_version: 1.0 nodata: -99 - version: 1.0 path: hydrography/reservoirs/reservoir-db.fgb rename: Depth_avg: Depth_avg @@ -1267,7 +1223,7 @@ koppen_geiger: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -1276,43 +1232,15 @@ koppen_geiger: paper_doi: 10.1127/0941-2948/2006/0130 paper_ref: Kottek et al. (2006) source_url: http://koeppen-geiger.vu-wien.ac.at/present.htm - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 + source_version: 2017 nodata: 0 - version: 2017 path: meteo/climate_classification_v2017/Map_KG-Global.tif -hydro_lake_atlas_pol: - crs: 4326 - data_type: GeoDataFrame - driver: vector - driver_kwargs: - layer: LakeATLAS_v10_pol - meta: - category: hydrography - notes: renaming and units might require some revision - paper_doi: 10.1038/s41597-022-01425-z - paper_ref: Lehner et al. (2022) - source_license: CC BY 4.0 - source_url: https://www.hydrosheds.org/hydroatlas - source_spatial_extent: - West: -180.0 - South: -55.865 - East: 180.0 - North: 83.576 - path: hydrography/hydro_atlas/lake_atlas_v10.gpkg - version: 10 - rename: - dis_m3_pyr: Dis_avg - mdt_cnes_cls18: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 720 y: 720 @@ -1321,21 +1249,17 @@ mdt_cnes_cls18: paper_doi: 10.5194/os-17-789-2021 paper_ref: Mulet et al (2021) source_url: https://www.aviso.altimetry.fr/en/data/products/auxiliary-products/mdt.html + source_version: 18 unit: m+GOCO05S - processing_script: topography/mdt_cnes_cls18/convert_lon_fill_mdt.ipynb - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 - version: 18 path: topography/mdt_cnes_cls18/MDT_CNES_CLS18_global_filled.tif merit: + alias: merit_v1.0.3 +merit_v1.0.3: crs: 4326 data_type: RasterDataset driver: netcdf - driver_kwargs: + kwargs: drop_variables: projection chunks: lat: 6000 @@ -1346,21 +1270,18 @@ merit: paper_ref: Yamazaki et al. (2018) source_license: CC-BY-NC 4.0 or ODbL 1.0 source_url: http://hydro.iis.u-tokyo.ac.jp/~yamadai/MERIT_DEM/ - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 - version: 1.0.3 + source_version: 1.0.3 path: topography/merit/merit.nc rename: elevation: elevtn merit_hydro: + alias: merit_hydro_v1.0 +merit_hydro_v1.0: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 6000 y: 6000 @@ -1368,16 +1289,9 @@ merit_hydro: category: topography paper_doi: 10.1029/2019WR024873 paper_ref: Yamazaki et al. (2019) - processing_notes: create vrt file with gdalbuildvrt; - additional variables for stream-order, basin index (basins), and slope (lndslp) derived with pyflwdir source_license: CC-BY-NC 4.0 or ODbL 1.0 source_url: http://hydro.iis.u-tokyo.ac.jp/~yamadai/MERIT_Hydro - source_spatial_extent: - West: -180.0 - South: -60.0 - East: 180.0 - North: 85.0 - version: 1.0 + source_version: 1.0 path: topography/merit_hydro/*.vrt rename: bas: basins @@ -1390,10 +1304,12 @@ merit_hydro: wth: rivwth merit_hydro_ihu: + alias: merit_hydro_ihu_30sec_v1.0 +merit_hydro_ihu_30sec_v1.0: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 6000 y: 6000 @@ -1404,14 +1320,8 @@ merit_hydro_ihu: source_license: ODC-By 1.0 source_url: https://zenodo.org/record/5166932#.YVbxJ5pByUk source_doi: 10.5281/zenodo.5166932 - processing_notes: topography/merit_hydro_ihu/README - source_spatial_extent: - West: -180.0 - South: -60.0 - East: 180.0 - North: 85.0 + source_version: 1.0 path: topography/merit_hydro_ihu/30sec/*.tif - version: 1.0 rename: 30sec_basids: basins 30sec_flwdir: flwdir @@ -1431,16 +1341,11 @@ merit_hydro_ihu_index: paper_ref: Eilander et al. (2021) source_license: CC-BY-NC 4.0 source_url: https://zenodo.org/record/5166932#.YVbxJ5pByUk - processing_notes: topography/merit_hydro_ihu/README - source_spatial_extent: - West: -180.025 - South: -59.467 - East: 190.35 - North: 83.683 - version: 1.0 path: topography/merit_hydro_ihu/30sec/basins.gpkg merit_hydro_index: + alias: merit_hydro_index_v1.0 +merit_hydro_index_v1.0: crs: 4326 data_type: GeoDataFrame driver: vector @@ -1449,19 +1354,15 @@ merit_hydro_index: paper_doi: 10.5194/hess-25-5287-2021 paper_ref: Eilander et al. (2021) source_license: CC-BY-NC 4.0 - source_spatial_extent: - West: -180.028 - South: -59.46 - East: 190.338 - North: 83.661 - version: 1.0 path: topography/merit_hydro/basin_index.fgb merit_hydro_patch: + alias: merit_hydro_patch_v1.0 +merit_hydro_patch_v1.0: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 6000 y: 6000 @@ -1471,14 +1372,7 @@ merit_hydro_patch: paper_ref: Yamazaki et al. (2019) source_license: CC-BY-NC 4.0 or ODbL 1.0 source_url: http://hydro.iis.u-tokyo.ac.jp/~yamadai/MERIT_Hydro - processing_script: topography/merit_hydro/patches/scripts - processing_notes: local corrections of flow direction raster and re-derive the related maps (basins, uparea, strord) - source_spatial_extent: - West: -180.0 - South: -60.0 - East: 180.0 - North: 85.0 - version: 1.0 + source_version: 1.0 Deltares patch path: topography/merit_hydro/patches/*.vrt rename: bas: basins @@ -1491,36 +1385,31 @@ merit_hydro_patch: wth: rivwth modis_lai: + alias: modis_mcd15a3h_v006 +modis_mcd15a3h_v006: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 concat: true meta: category: landuse + notes: this dataset has been extracted from GEE ('MODIS/006/MCD15A3H') for the period '2003-01-01', '2017-12-31' paper_doi: 10.5067/MODIS/MCD15A3H.006 paper_ref: Myneni et al (2015) source_license: https://lpdaac.usgs.gov/data/data-citation-and-policies/ source_url: https://lpdaac.usgs.gov/products/mcd15a3hv006/ - processing_notes: this dataset has been extracted from GEE ('MODIS/006/MCD15A3H') - for the period '2003-01-01', '2017-12-31' - processing_script: - GEE_script: landuse/modis/MODIS_MCD15A3H_LAI/GEE_MODIS_LAI.js - merge_script: landuse/modis/MODIS_MCD15A3H_LAI/merge_rasters_GEE_LAI.py - source_spatial_extent: - West: -180.004 - South: -90.002 - East: 180.004 - North: 90.002 - version: MCD15A3H V006 + source_version: MCD15A3H V006 path: landuse/modis/MODIS_MCD15A3H_LAI/*.tif unit_mult: LAI: 0.1 osm_coastlines: + alias: osm_coastlines_v20200109 +osm_coastlines_v20200109: crs: 4326 data_type: GeoDataFrame driver: vector @@ -1530,17 +1419,14 @@ osm_coastlines: source_author: OpenStreetMap source_license: ODbL source_url: https://osmdata.openstreetmap.de/data/coastlines.html - source_spatial_extent: - West: -180.0 - South: -78.733 - East: 180.0 - North: 90.0 - version: 1.0 + source_version: 1.0 path: geography/osm/osm_coastlines-db.fgb rename: fid: coastline_id osm_landareas: + alias: osm_landareas_v20200109 +osm_landareas_v20200109: crs: 4326 data_type: GeoDataFrame driver: vector @@ -1550,12 +1436,7 @@ osm_landareas: source_author: OpenStreetMap source_license: ODbL source_url: https://osmdata.openstreetmap.de/data/coastlines.html - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 83.666 - version: 1.0 + source_version: 1.0 path: geography/osm/osm_landareas-db.fgb rename: fid: land_id @@ -1572,12 +1453,8 @@ pcr_globwb: paper_doi: 10.5281/zenodo.1045339 paper_ref: Sutanudjaja, E. H., et al (2017) source_url: https://zenodo.org/records/1045339#.XWUr7E2P5aR + source_version: 2017.11b1 processing_script: hydro/pcr_globwb/prep_glob.py - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 variants: - version: 1990 path: hydro/pcr_globwb/glob_1990.zarr @@ -1604,6 +1481,8 @@ pcr_globwb: lsk_gross: 1000 rgi: + alias: rgi_v0.6 +rgi_v0.6: crs: 4326 data_type: GeoDataFrame driver: vector @@ -1614,12 +1493,7 @@ rgi: paper_ref: Pfeffer et al. (2014) source_license: CC BY 4.0 source_url: https://cds.climate.copernicus.eu/cdsapp#!/dataset/insitu-glaciers-extent?tab=overview - source_spatial_extent: - West: -179.921 - South: -78.309 - East: 179.751 - North: 83.608 - version: 6.0 + source_version: 6.0 path: hydrography/rgi/rgi.fgb rename: C3S_ID: C3S_id @@ -1627,28 +1501,7 @@ rgi: ID: simple_id RGIID: RGI_id -hydro_river_atlas: - crs: 4326 - data_type: GeoDataFrame - driver: vector - meta: - category: hydrography - notes: renaming and units might require some revision - paper_doi: 10.1038/s41597-019-0300-6 - paper_ref: Linke et al. (2019) - source_license: CC BY 4.0 - source_url: https://www.hydrosheds.org/hydroatlas - source_spatial_extent: - West: -179.998 - South: -55.877 - East: 179.998 - North: 83.59 - path: hydrography/hydro_atlas/river_atlas_v10.gpkg - version: 10 - rename: - dis_m3_pyr: Dis_avg - -hydro_rivers_lin2019: +rivers_lin2019_v1: data_type: GeoDataFrame driver: vector meta: @@ -1657,23 +1510,19 @@ hydro_rivers_lin2019: paper_ref: Lin et al. (2019) source_license: CC-BY-NC 4.0 source_url: https://zenodo.org/record/3552776#.YVbOrppByUk - processing_notes: hydrography/rivers_lin2019/README - source_spatial_extent: - West: -177.204 - South: -55.414 - East: 180.237 - North: 82.112 - version: 1 + source_version: 1 path: hydrography/rivers_lin2019/rivers_ge30m.fgb rename: width_m: rivwth Q2: qbankfull simard: + alias: simard_v2011 +simard_v2011: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -1682,103 +1531,69 @@ simard: paper_doi: 10.1029/2011JG001708 paper_ref: Simard et al (2011) source_url: https://webmap.ornl.gov/ogc/dataset.jsp?ds_id=10023 - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 - version: 2011 path: landuse/simard/sdat_10023_canopy_height_simard.tif -SM2RAIN_ASCAT_monthly_025: +SM2RAIN_ASCAT_monthly_025_v1.4: crs: 4326 data_type: RasterDataset driver: netcdf - driver_kwargs: + kwargs: chunks: latitude: 360 longitude: 720 time: 84 meta: category: meteo - notes: crs guessed as it is neither mentioned in the data nor in the literature, - 0.25-degree resolution + notes: crs guessed as it is neither mentioned in the data nor in the literature, 0.25-degree resolution paper_doi: 10.5194/essd-11-1583-2019 paper_ref: Brocca et al. (2019) source_license: https://creativecommons.org/licenses/by/4.0/legalcode source_url: https://zenodo.org/record/4570192#.YueKJWNByUl - source_temporal_extent: - start: '2007-01-01' - end: '2020-12-01' - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 - version: 1.4 path: meteo/sm2rain_ascat/SM2RAIN_ASCAT_monthly_025_2007_2020.nc rename: rainfall: precip -SM2RAIN_ASCAT_monthly_05: +SM2RAIN_ASCAT_monthly_05_v1.4: crs: 4326 data_type: RasterDataset driver: netcdf - driver_kwargs: + kwargs: chunks: latitude: 180 longitude: 360 time: 84 meta: category: meteo - notes: crs guessed as it is neither mentioned in the data nor in the literature, - 0.5-degree resolution + notes: crs guessed as it is neither mentioned in the data nor in the literature, 0.5-degree resolution paper_doi: 10.5194/essd-11-1583-2019 paper_ref: Brocca et al. (2019) source_license: https://creativecommons.org/licenses/by/4.0/legalcode source_url: https://zenodo.org/record/4570192#.YueKJWNByUl - source_temporal_extent: - start: '2007-01-01' - end: '2020-12-01' - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 - version: 1.4 path: meteo/sm2rain_ascat/SM2RAIN_ASCAT_monthly_05_2007_2020.nc rename: rainfall: precip soilgrids: + alias: soilgrids_v1.0 +soilgrids_v1.0: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 2400 y: 2400 meta: category: soil - notes: "soilthickness is based on 1) soilgrids (global) and 2) dataset for Eurasia\ - \ (ESDAC, 2004; Panagos et al., 2012): ESDAC, 2004. The european soil database\ - \ distribution version 2.0, european commission and the European soil bureau\n\ - \ network. esdac.jrc.ec.europa.eu, accessed: 2017-11-17. Panagos, P., Van\ - \ Liedekerke, M., Jones, A., Montanarella, L., 2012. European soil data centre:\ - \ Response to european policy support and public data requirements. Land\ - \ Use Policy 29 (2), 329\xC3\xA2\xE2\u201A\xAC\xE2\u20AC\u0153338. \n" + notes: "soilthickness is based on 1) soilgrids (global) and 2) dataset for Eurasia (ESDAC, 2004; Panagos et al., 2012): ESDAC, 2004. The european soil + database distribution version 2.0, european commission and the European soil bureau\n network. esdac.jrc.ec.europa.eu, accessed: 2017-11-17. Panagos, + P., Van Liedekerke, M., Jones, A., Montanarella, L., 2012. European soil data centre: Response to european policy support and public data requirements. + Land Use Policy 29 (2), 329–338. \n" paper_doi: 10.1371/journal.pone.0169748 paper_ref: Hengl et al. (2017) source_license: ODbL - source_url: https://www.isric.org/explore/soilgrids/faq-soilgrids-201 - processing_notes: "soilthickness is based on 1) soilgrids (global, depth to bedrock - BDRICM variable) and 2) dataset for Eurasia" - processing_script: p:/wflow_global/static_data/wflow_sbm_parameters/ - source_spatial_extent: - West: -19949000.0 - South: -6147500.0 - East: 19861750.0 - North: 8361000.0 - version: 1.0 + source_url: https://www.isric.org/explore/soilgrids/faq-soilgrids-2017 + source_version: 2017 path: soil/soilgrids_v1.0/*_250m_ll.tif rename: BLDFIE_M_sl1_250m_ll: bd_sl1 @@ -1848,11 +1663,117 @@ soilgrids: ph_sl6: 0.1 ph_sl7: 0.1 -vito_2015: +soilgrids_2020: + alias: soilgrids_v2.0 +soilgrids_v2.0: + data_type: RasterDataset + driver: raster + kwargs: + chunks: + x: 2400 + y: 2400 + meta: + category: soil + notes: "soilthickness is based on 1) soilgrids (global) and 2) dataset for Eurasia (ESDAC, 2004; Panagos et al., 2012): ESDAC, 2004. The european soil + database distribution version 2.0, european commission and the European soil bureau network. esdac.jrc.ec.europa.eu, accessed: 2017-11-17. Panagos, + P., Van Liedekerke, M., Jones, A., Montanarella, L., 2012. European soil data centre: Response to european policy support and public data requirements. + Land Use Policy 29 (2), 329–338. \n" + paper_doi: https://doi.org/10.5194/soil-2020-65 + paper_ref: de Sousa et al. (2020) + source_license: CC BY 4.0 + source_url: https://www.isric.org/explore/soilgrids/faq-soilgrids + source_version: 2020 + path: soil/soilgrids_v2.0/*/*_mean.vrt + rename: + bdod_0-5cm_mean: bd_sl1 + bdod_5-15cm_mean: bd_sl2 + bdod_15-30cm_mean: bd_sl3 + bdod_30-60cm_mean: bd_sl4 + bdod_60-100cm_mean: bd_sl5 + bdod_100-200cm_mean: bd_sl6 + clay_0-5cm_mean: clyppt_sl1 + clay_5-15cm_mean: clyppt_sl2 + clay_15-30cm_mean: clyppt_sl3 + clay_30-60cm_mean: clyppt_sl4 + clay_60-100cm_mean: clyppt_sl5 + clay_100-200cm_mean: clyppt_sl6 + soc_0-5cm_mean: oc_sl1 + soc_5-15cm_mean: oc_sl2 + soc_15-30cm_mean: oc_sl3 + soc_30-60cm_mean: oc_sl4 + soc_60-100cm_mean: oc_sl5 + soc_100-200cm_mean: oc_sl6 + phh2o_0-5cm_mean: ph_sl1 + phh2o_5-15cm_mean: ph_sl2 + phh2o_15-30cm_mean: ph_sl3 + phh2o_30-60cm_mean: ph_sl4 + phh2o_60-100cm_mean: ph_sl5 + phh2o_100-200cm_mean: ph_sl6 + silt_0-5cm_mean: sltppt_sl1 + silt_5-15cm_mean: sltppt_sl2 + silt_15-30cm_mean: sltppt_sl3 + silt_30-60cm_mean: sltppt_sl4 + silt_60-100cm_mean: sltppt_sl5 + silt_100-200cm_mean: sltppt_sl6 + sand_0-5cm_mean: sndppt_sl1 + sand_5-15cm_mean: sndppt_sl2 + sand_15-30cm_mean: sndppt_sl3 + sand_30-60cm_mean: sndppt_sl4 + sand_60-100cm_mean: sndppt_sl5 + sand_100-200cm_mean: sndppt_sl6 + SoilThickness_250_mean: soilthickness + TAXOUSDA_250_mean: tax_usda + unit_mult: + bd_sl1: 0.01 + bd_sl2: 0.01 + bd_sl3: 0.01 + bd_sl4: 0.01 + bd_sl5: 0.01 + bd_sl6: 0.01 + bd_sl7: 0.01 + oc_sl1: 0.01 + oc_sl2: 0.01 + oc_sl3: 0.01 + oc_sl4: 0.01 + oc_sl5: 0.01 + oc_sl6: 0.01 + oc_sl7: 0.01 + ph_sl1: 0.1 + ph_sl2: 0.1 + ph_sl3: 0.1 + ph_sl4: 0.1 + ph_sl5: 0.1 + ph_sl6: 0.1 + ph_sl7: 0.1 + clyppt_sl1: 0.1 + clyppt_sl2: 0.1 + clyppt_sl3: 0.1 + clyppt_sl4: 0.1 + clyppt_sl5: 0.1 + clyppt_sl6: 0.1 + clyppt_sl7: 0.1 + sltppt_sl1: 0.1 + sltppt_sl2: 0.1 + sltppt_sl3: 0.1 + sltppt_sl4: 0.1 + sltppt_sl5: 0.1 + sltppt_sl6: 0.1 + sltppt_sl7: 0.1 + sndppt_sl1: 0.1 + sndppt_sl2: 0.1 + sndppt_sl3: 0.1 + sndppt_sl4: 0.1 + sndppt_sl5: 0.1 + sndppt_sl6: 0.1 + sndppt_sl7: 0.1 + +vito: + alias: vito_2015_v2.0.2 +vito_2015_v2.0.2: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -1861,19 +1782,14 @@ vito_2015: paper_doi: 10.5281/zenodo.3939038 paper_ref: Buchhorn et al (2020) source_url: https://land.copernicus.eu/global/products/lc - source_spatial_extent: - West: -180.0 - South: -60.0 - East: 180.0 - North: 80.0 - version: 2.0.2 + source_version: v2.0.2 path: landuse/vito/ProbaV_LC100_epoch2015_global_v2.0.2_discrete-classification_EPSG-4326.tif -vito_2016: +vito_2016_v3.0.1: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -1882,19 +1798,14 @@ vito_2016: paper_doi: 10.5281/zenodo.3518026 paper_ref: Buchhorn et al (2020) source_url: https://land.copernicus.eu/global/products/lc - source_spatial_extent: - West: -180.0 - South: -60.0 - East: 180.0 - North: 80.0 - version: 3.0.1 + source_version: v3.0.1 path: landuse/vito/PROBAV_LC100_global_v3.0.1_2016-conso_Discrete-Classification-map_EPSG-4326.tif -vito_2017: +vito_2017_v3.0.1: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -1903,19 +1814,14 @@ vito_2017: paper_doi: 10.5281/zenodo.3518036 paper_ref: Buchhorn et al (2020) source_url: https://land.copernicus.eu/global/products/lc - source_spatial_extent: - West: -180.0 - South: -60.0 - East: 180.0 - North: 80.0 - version: 3.0.1 + source_version: v3.0.1 path: landuse/vito/PROBAV_LC100_global_v3.0.1_2017-conso_Discrete-Classification-map_EPSG-4326.tif -vito_2018: +vito_2018_v3.0.1: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -1924,19 +1830,14 @@ vito_2018: paper_doi: 10.5281/zenodo.3518038 paper_ref: Buchhorn et al (2020) source_url: https://land.copernicus.eu/global/products/lc - source_spatial_extent: - West: -180.0 - South: -60.0 - East: 180.0 - North: 80.0 - version: 3.0.1 + source_version: v3.0.1 path: landuse/vito/PROBAV_LC100_global_v3.0.1_2018-conso_Discrete-Classification-map_EPSG-4326.tif -vito_2019: +vito_2019_v3.0.1: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 3600 y: 3600 @@ -1945,15 +1846,12 @@ vito_2019: paper_doi: 10.5281/zenodo.3939050 paper_ref: Buchhorn et al (2020) source_url: https://land.copernicus.eu/global/products/lc - source_spatial_extent: - West: -180.0 - South: -60.0 - East: 180.0 - North: 80.0 - version: 3.0.1 + source_version: v3.0.1 path: landuse/vito/ProbaV_LC100_global_v3.0.1_2019-nrt_discrete-classification-map_EPSG-4326.tif wb_countries: + alias: wb_countries_v20200319 +wb_countries_v20200319: crs: 4326 data_type: GeoDataFrame driver: vector @@ -1963,12 +1861,6 @@ wb_countries: source_url: https://datacatalog.worldbank.org/dataset/world-bank-official-boundaries source_license: CC-BY 4.0 timestamp: February 2020 - source_spatial_extent: - West: -180.0 - South: -59.473 - East: 180.0 - North: 83.634 - version: 20200319 path: geography/wb/WB_countries_Admin0.fgb rename: ISO_A3: country_iso @@ -1980,10 +1872,12 @@ wb_countries: gdp_pc: 1000 worldclim: + alias: worldclim_v2.0 +worldclim_v2.0: crs: 4326 data_type: RasterDataset driver: netcdf - driver_kwargs: + kwargs: chunks: lat: 3600 lon: 3600 @@ -1992,13 +1886,8 @@ worldclim: paper_doi: 10.1002/joc.5086 paper_ref: Fick and Hijmans (2017) source_url: https://www.worldclim.org/data/worldclim21.html + source_version: 2 nodata: -999.0 - source_spatial_extent: - West: -180.0 - South: -90.0 - East: 180.0 - North: 90.0 - version: 2.0 path: meteo/worldclim_v2.0/wc2.0_30s_prec.nc rename: prec: precip @@ -2006,7 +1895,7 @@ worldclim: worldpop_2020_constrained: data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: auto meta: category: socio-economic @@ -2014,38 +1903,24 @@ worldpop_2020_constrained: paper_ref: Stevens et al. (2015) source_license: CC BY 4.0 source_url: https://www.worldpop.org/doi/10.5258/SOTON/WP00684 - processing_script: socio_economic/worldpop/src/ - source_spatial_extent: - West: -180.0 - South: -55.985 - East: 180.0 - North: 83.628 nodata: -99999 path: socio_economic/worldpop/ppp_2020_constrained.vrt rename: ppp_2020_constrained: population worldpop_2020_UNadj_constrained: - data_type: RasterDataset - driver: raster - driver_kwargs: - chunks: auto - meta: - processing_script: socio_economic/worldpop/src/ - source_spatial_extent: - West: -180.0 - South: -55.985 - East: 180.0 - North: 83.628 + alias: worldpop_2020_constrained path: socio_economic/worldpop/ppp_2020_UNadj_constrained.vrt rename: ppp_2020_UNadj_constrained: population wsf_bld_2015: + alias: wsf_bld_2015_v1 +wsf_bld_2015_v1: crs: 4326 data_type: RasterDataset driver: raster - driver_kwargs: + kwargs: chunks: x: 1000 y: 1000 @@ -2055,10 +1930,4 @@ wsf_bld_2015: paper_ref: Marconcini at al (2020) source_license: CC0 1.0 source_url: https://un-spider.org/links-and-resources/data-sources/world-settlement-footprint-2015-wsf-dlr-eoc - source_spatial_extent: - West: -180.0 - South: -60.0 - East: 180.0 - North: 80.0 - version: 1 path: infrastructure/wsf/WSF2015_v1_EPSG4326.vrt diff --git a/data/predefined_catalogs.yml b/data/predefined_catalogs.yml index c96d9eadd..f7cae9857 100644 --- a/data/predefined_catalogs.yml +++ b/data/predefined_catalogs.yml @@ -16,8 +16,7 @@ deltares_data: urlpath: https://raw.githubusercontent.com/Deltares/hydromt/{version}/data/catalogs/deltares_data.yml versions: - v2024.2: main - v2024.1.30: 503f9a400e8df89ecca240091d8eba9de1fedf75 + v2024.1.30: main v2023.12.21: 392206b21b26e62e00ae76db7ffc61a3b95e2175 v2023.12.19: bf25e79dcbda67612a75588cd782d57abe3881de v2023.2: 0bf8e2a1e716095dc6df54a5e9114ce88da0701b @@ -37,8 +36,9 @@ aws_data: v2023.2: 897e5c5272875f1c066f393798b7ae59721c9e9d notes: This data are stored in public Amazon Web Services. artifact_data: - urlpath: https://raw.githubusercontent.com/Deltares/hydromt/{version}/data/catalogs/artifact_data.yml + urlpath: https://github.com/DirkEilander/hydromt-artifacts/releases/download/{version}/data.tar.gz versions: - v0.0.9: main - v0.0.8: 202874eb4fe3415d0608ea81cd61620af6f5816a + v0.0.8: v0.0.8 + v0.0.7: v0.0.7 + v0.0.6: v0.0.6 notes: This data archive contains a sample dataset for the Piave basin (North Italy) to be used for tests and docs/demo purposes. diff --git a/docs/changelog.rst b/docs/changelog.rst index 645b1320b..db57a3c72 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -37,11 +37,6 @@ Added - Test script for testing predefined catalogs locally. (#735) - Option to write a data catalog to a csv file (#425) -Changed -------- -- Datacatalog preserves variant specific meta data (#521) -- Updated DataCatalogValidator to deal with provider and driver_kwargs (#521) - Fixed ----- - Reading Vector formats that consist of more than one file via geopandas. (#691) @@ -50,7 +45,6 @@ Fixed - Fix bug in `raster._check_dimensions` for datasets with multiple variables with varying dimension size (#761) - Fix bug when reading COGs at requested zoom level (#758) - v0.9.2 (2024-01-09) =================== This release adds additional bug fixes for the meridian offset functinality, and improvements to the new CLI commands. @@ -61,11 +55,9 @@ Added - New stats.skills VE and RSR (#666) - Check CLI command can now validate bbox and geom regions (#664) - Changed ------- - Export CLI now uses '-s' for source, '-t' for time and '-i' for config. (#660) -- Updated deltares_data data catalog to incorporate the newest data catalog features (#667) Fixed ----- diff --git a/examples/delineate_basin.ipynb b/examples/delineate_basin.ipynb index 17b245fdc..e674c76dc 100644 --- a/examples/delineate_basin.ipynb +++ b/examples/delineate_basin.ipynb @@ -33,7 +33,8 @@ "source": [ "import numpy as np\n", "import geopandas as gpd\n", - "from shapely.geometry import box" + "from shapely.geometry import box\n", + "from pprint import pprint" ] }, { @@ -377,7 +378,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.10.8" }, "vscode": { "interpreter": { diff --git a/examples/prep_data_catalog.ipynb b/examples/prep_data_catalog.ipynb index 9f1d16fdd..c74066a24 100644 --- a/examples/prep_data_catalog.ipynb +++ b/examples/prep_data_catalog.ipynb @@ -64,7 +64,7 @@ "outputs": [], "source": [ "# this download the artifact_data archive v0.0.6\n", - "data_catalog = hydromt.DataCatalog(data_libs=[\"artifact_data=v0.0.8\"])\n", + "data_catalog = hydromt.DataCatalog(data_libs=[\"artifact_data=v0.0.6\"])\n", "pprint(data_catalog.predefined_catalogs)" ] }, @@ -85,7 +85,7 @@ "outputs": [], "source": [ "# the artifact data is stored in the following location\n", - "root = os.path.join(data_catalog._cache_dir, \"artifact_data\", \"v0.0.8\")\n", + "root = os.path.join(data_catalog._cache_dir, \"artifact_data\", \"v0.0.6\")\n", "# let's print some of the file that are there\n", "for item in os.listdir(root)[-10:]:\n", " print(item)" diff --git a/hydromt/data_adapter/data_adapter.py b/hydromt/data_adapter/data_adapter.py index 2ea21731e..9c9314974 100644 --- a/hydromt/data_adapter/data_adapter.py +++ b/hydromt/data_adapter/data_adapter.py @@ -334,13 +334,9 @@ def _resolve_paths( continue key_str = "{" + f"{key}:{fmt}" + "}" if fmt else "{" + key + "}" # remove unused fields - if ( - key in ["year", "month"] - and time_tuple is None - and not path.endswith("*") - ): + if key in ["year", "month"] and time_tuple is None: path += "*" - elif key == "variable" and variables is None and not path.endswith("*"): + elif key == "variable" and variables is None: path += "*" # escape unknown fields elif key is not None and key not in known_keys: diff --git a/hydromt/data_catalog.py b/hydromt/data_catalog.py index d306daf23..9402c31d3 100644 --- a/hydromt/data_catalog.py +++ b/hydromt/data_catalog.py @@ -1913,9 +1913,6 @@ def _denormalise_data_dict(data_dict) -> List[Tuple[str, Dict]]: variants = source.pop("variants") for diff in variants: source_copy = copy.deepcopy(source) - if "meta" in source_copy: - # Update meta if variant has variant specific meta data - source_copy["meta"].update(diff.pop("meta", {})) source_copy.update(**diff) data_dicts.append({name: source_copy}) elif "placeholders" in source: diff --git a/hydromt/validators/data_catalog.py b/hydromt/validators/data_catalog.py index 9e6391146..a5592d314 100644 --- a/hydromt/validators/data_catalog.py +++ b/hydromt/validators/data_catalog.py @@ -111,9 +111,7 @@ class DataCatalogItem(BaseModel): path: Optional[Path] = None crs: Optional[Union[int, str]] = None filesystem: Optional[str] = None - provider: Optional[str] = None - driver_kwargs: Dict[str, Any] = Field(default_factory=dict) - kwargs: Dict[str, Any] = Field(default_factory=dict) # deprecated + kwargs: Dict[str, Any] = Field(default_factory=dict) storage_options: Dict[str, Any] = Field(default_factory=dict) placeholders: Optional[Dict[str, Any]] = None rename: Dict[str, str] = Field(default_factory=dict) diff --git a/tests/data/merged_esa_worldcover.yml b/tests/data/merged_esa_worldcover.yml index 88143b1b1..7f7778587 100644 --- a/tests/data/merged_esa_worldcover.yml +++ b/tests/data/merged_esa_worldcover.yml @@ -18,8 +18,6 @@ esa_worldcover: - provider: local version: 2020 path: landuse/esa_worldcover/esa-worldcover.vrt - meta: - source_author: Test Dummy - provider: aws version: 2020 path: s3://esa-worldcover/v100/2020/ESA_WorldCover_10m_2020_v100_Map_AWS.vrt diff --git a/tests/test_data_catalog.py b/tests/test_data_catalog.py index 8682982ae..958ffb97a 100644 --- a/tests/test_data_catalog.py +++ b/tests/test_data_catalog.py @@ -195,12 +195,6 @@ def test_versioned_catalog_entries(tmpdir): # test round trip to and from dict merged_catalog2 = DataCatalog().from_dict(merged_catalog.to_dict()) assert merged_catalog2 == merged_catalog - # test updating source meta data from version meta data - source_2020_local = merged_catalog.get_source( - "esa_worldcover", version="2020", provider="local" - ) - assert source_2020_local.meta["source_author"] == "Test Dummy" - assert source_2020_local.meta["source_license"] == "CC BY 4.0" # Make sure we can query for the version we want aws_and_legacy_catalog = DataCatalog(data_libs=[legacy_yml_fn, aws_yml_fn]) @@ -594,12 +588,12 @@ def test_get_dataframe(df, tmpdir): def test_deprecation_warnings(artifact_data): with pytest.deprecated_call(): - # should be DataCatalog(data_libs=['v0.0.8']) - DataCatalog(artifact_data="v0.0.8") + # should be DataCatalog(data_libs=['artifact_data=v0.0.6']) + DataCatalog(artifact_data="v0.0.6") with pytest.deprecated_call(): cat = DataCatalog() - # should be cat.from_predefined_catalogs('artifact_data', 'v0.0.8') - cat.from_artifacts("artifact_data", version="v0.0.8") + # should be cat.from_predefined_catalogs('artifact_data', 'v0.0.6') + cat.from_artifacts("artifact_data", version="v0.0.6") with pytest.deprecated_call(): fn = artifact_data["chelsa"].path # should be driver_kwargs=dict(chunks={'x': 100, 'y': 100})