diff --git a/docs/getting_started_guide/providers.rst b/docs/getting_started_guide/providers.rst index d3bc844e7..9d7e7349e 100644 --- a/docs/getting_started_guide/providers.rst +++ b/docs/getting_started_guide/providers.rst @@ -13,6 +13,7 @@ Products from the following providers are made avaiable through ``eodag``: * `cop_ads `_: Copernicus Atmosphere Data Store * `cop_cds `_: Copernicus Climate Data Store * `cop_dataspace `_: Copernicus Data Space +* `cop_ewds `_: CEMS Early Warning Data Store * `cop_marine `_: Copernicus Marine Service * `creodias `_: CloudFerro DIAS * `creodias_s3 `_: CloudFerro DIAS data through S3 protocol diff --git a/docs/getting_started_guide/register.rst b/docs/getting_started_guide/register.rst index 03317eac6..fb8e4cb8e 100644 --- a/docs/getting_started_guide/register.rst +++ b/docs/getting_started_guide/register.rst @@ -64,8 +64,8 @@ Then log in and go to your user profile on `Climate Data Store `__ +``cop_ewds`` +^^^^^^^^^^^^ +Go to the `ECMWF homepage `__ and create an account by clicking on *Log in* and then *Register*. +Then log in and go to your user profile on `CEMS Early Warning Data Store `__ and use your +*Personal Access Token* as *apikey* in eodag credentials. + +To download data, you also have to accept certain terms depending on the dataset. There are two different licences that have to be accepted +to use the CEMS EWDS datasets. Accepting the `CEMS-FLOODS datasets licence` is necessary to use the `GLOFAS` and `EFAS` datasets, +the `Licence to use Copernicus Products` is valid for the Fire danger datasets. + +To accept a licence: + +* Search for the dataset you want to download `here `__ while being + logged in. +* Open the dataset details and go to the download tab. +* Scroll down and accept the licence in the section `Terms of use`. +* You can check which licences you have accepted in your user profile. + ``cop_marine`` ^^^^^^^^^^^^^^ No account is required diff --git a/docs/index.rst b/docs/index.rst index 68e9ecc5b..69f45e9ab 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,7 +30,8 @@ types (Sentinel 1, Sentinel 2, Sentinel 3, Landsat, etc.) that can be searched a `aws_eos `_, `cop_ads `_, `cop_cds `_, - `cop_dataspace `_, + `cop_dataspace `_, + `cop_ewds `_, `cop_marine `_, `creodias `_, `creodias_s3 `_, diff --git a/docs/notebooks/tutos/tuto_cds.ipynb b/docs/notebooks/tutos/tuto_cds.ipynb index 5cc8d0c4b..a6f080863 100644 --- a/docs/notebooks/tutos/tuto_cds.ipynb +++ b/docs/notebooks/tutos/tuto_cds.ipynb @@ -6,7 +6,7 @@ "source": [ "# Copernicus Atmosphere using BuildSearchResult plugin\n", "\n", - "In this tutorial we will show you how to use eodag to download data from providers using [BuildSearchResult](https://eodag.readthedocs.io/en/latest/plugins_reference/generated/eodag.plugins.search.build_search_result.BuildSearchResult.html) `eodag` plugin. You can currently find two providers that uses it, `cop_ads` and `cop_cds`. For this tutorial we will use `cop_ads`, but `cop_cds` is used the same way." + "In this tutorial we will show you how to use eodag to download data from providers using [BuildSearchResult](https://eodag.readthedocs.io/en/latest/plugins_reference/generated/eodag.plugins.search.build_search_result.BuildSearchResult.html) `eodag` plugin. You can currently find three providers that uses it, `cop_ads`, `cop_cds` and `cop_ewds`. For this tutorial we will use `cop_ads`, but `cop_cds` and `cop_ewds` are used the same way." ] }, { diff --git a/eodag/resources/product_types.yml b/eodag/resources/product_types.yml index aa2a6dc8b..5424e48ba 100644 --- a/eodag/resources/product_types.yml +++ b/eodag/resources/product_types.yml @@ -3689,6 +3689,39 @@ FIRE_HISTORICAL: title: Fire danger indices historical data from the Copernicus Emergency Management Service missionStartDate: "1940-01-03T00:00:00Z" +FIRE_SEASONAL: + abstract: | + This dataset offers modeled daily fire danger time series, driven by seasonal weather forecasts. It provides long-range + predictions of meteorological conditions conducive to the initiation, spread, and persistence of fires. The fire danger + metrics included in this dataset are part of an extensive dataset produced by the Copernicus Emergency Management Service (CEMS) + for the European Forest Fire Information System (EFFIS) and the Global Wildfire Information System (GWIS). EFFIS and GWIS + are used for monitoring and forecasting fire danger at both European and global scales. The dataset incorporates fire danger + indices from the U.S. Forest Service National Fire-Danger Rating System (NFDRS), the Canadian Forest Service Fire Weather Index Rating System (FWI), + and the Australian McArthur (Mark 5) rating systems. + This dataset was generated by driving the Global ECMWF Fire Forecast (GEFF) model with seasonal weather ensemble forecasts + from the European Centre for Medium-Range Weather Forecasts (ECMWF) System 5 (SEAS5) prediction system.These forecasts initially + consist of 25 ensemble members until December 2016, referred to as re-forecasts. After that period, they consist of seasonal + forecasts with 51 members. It is important to note that the re-forecast dataset was initialized using ERA-Interim analysis data, + while forecast simulations from 2016 onward are initialized using ECMWF operational analysis. Therefore, it is suggested that + the period 1981-2016 be used as a reference period, while the period 2017-to present as a real time forecast. + For both the re-forecast (1981-2016) and forecast periods (2017-present), the temporal resolution is daily forecasts at 12:00 + local time, available once a month, with a prediction horizon of 216 days (equivalent to 7 months). The data records in this + dataset will be extended over time as seasonal forcing data becomes available. Once the SEAS5 operation ceases, the dataset + will be updated with the next ECMWF seasonal system (SEAS6). It is essential to note that this is not a real-time service, + as real-time forecasts are accessible through the EFFIS web services. + These seasonal forecasts can be used to assess the performance of the forecasting system or to develop tools for statistically + correcting forecast errors. ECMWF produces this dataset as the computational center for fire danger forecasting within the + Copernicus Emergency Management Service (CEMS) on behalf of the Joint Research Centre, which serves as the managing entity for this service. + instrument: + platform: CEMS + platformSerialIdentifier: CEMS + processingLevel: + keywords: ECMWF,CEMS,EFFIS,GWIS,fire,danger,seasonal,GEFF + sensorType: + license: proprietary + title: Seasonal forecast of fire danger indices from the Copernicus Emergency Management Service + missionStartDate: "1981-02-01T00:00:00Z" + GLOFAS_FORECAST: abstract: | This dataset contains global modelled daily data of river discharge forced with meteorological forecasts. diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml index 51226cf66..9b53651c2 100644 --- a/eodag/resources/providers.yml +++ b/eodag/resources/providers.yml @@ -2574,15 +2574,6 @@ "time": {startTimeFromAscendingNode#get_ecmwf_time} }} - '{$.completionTimeFromAscendingNode#to_iso_date}' - anchor_hday_hmonth_hyear: &hday_hmonth_hyear - completionTimeFromAscendingNode: - - | - {{ - "hyear": {_date#interval_to_datetime_dict}["year"], - "hmonth": {_date#interval_to_datetime_dict}["month"], - "hday": {_date#interval_to_datetime_dict}["day"] - }} - - '{$.completionTimeFromAscendingNode#to_iso_date}' auth: !plugin type: HTTPHeaderAuth matching_url: https://cds-beta.climate.copernicus.eu @@ -2639,33 +2630,6 @@ GLACIERS_DIST_RANDOLPH: output_extension: .zip extract: true - FIRE_HISTORICAL: - output_extension: .grib - GLOFAS_FORECAST: - output_extension: .grib - GLOFAS_REFORECAST: - output_extension: .grib - GLOFAS_HISTORICAL: - output_extension: .grib - GLOFAS_SEASONAL: - output_extension: .grib - GLOFAS_SEASONAL_REFORECAST: - output_extension: .grib - EFAS_FORECAST: - output_extension: .grib.zip - extract: true - EFAS_HISTORICAL: - output_extension: .grib.zip - extract: true - EFAS_REFORECAST: - output_extension: .grib.zip - extract: true - EFAS_SEASONAL: - output_extension: .grib.zip - extract: true - EFAS_SEASONAL_REFORECAST: - output_extension: .grib.zip - extract: true SATELLITE_CARBON_DIOXIDE: output_extension: .zip extract: true @@ -2947,128 +2911,6 @@ "hydrological_year": {completionTimeFromAscendingNode#get_hydrological_year} }} - '{$.completionTimeFromAscendingNode#get_hydrological_year}' - FIRE_HISTORICAL: - dataset: cems-fire-historical-v1 - grid: original_grid - dataset_type: consolidated_dataset - api_product_type: reanalysis - variable: build_up_index - system_version: '4_1' - format: grib - metadata_mapping: - <<: *day_month_year - GLOFAS_FORECAST: - dataset: cems-glofas-forecast - system_version: operational - variable: river_discharge_in_the_last_24_hours - format: grib2 - hydrological_model: htessel_lisflood - api_product_type: control_forecast - leadtime_hour: '24' - download_format: zip - metadata_mapping: - <<: *day_month_year - GLOFAS_REFORECAST: - dataset: cems-glofas-reforecast - variable: river_discharge_in_the_last_24_hours - format: grib2 - download_format: zip - system_version: version_4_0 - hydrological_model: lisflood - api_product_type: control_reforecast - leadtime_hour: '24' - metadata_mapping: - <<: *hday_hmonth_hyear - GLOFAS_HISTORICAL: - dataset: cems-glofas-historical - system_version: version_4_0 - variable: river_discharge_in_the_last_24_hours - format: grib2 - download_format: zip - hydrological_model: lisflood - api_product_type: consolidated - metadata_mapping: - <<: *hday_hmonth_hyear - GLOFAS_SEASONAL: - dataset: cems-glofas-seasonal - variable: river_discharge_in_the_last_24_hours - format: grib2 - download_format: zip - system_version: operational - hydrological_model: lisflood - leadtime_hour: '24' - metadata_mapping: - <<: *day_month_year - GLOFAS_SEASONAL_REFORECAST: - dataset: cems-glofas-seasonal-reforecast - format: grib2 - download_format: zip - variable: river_discharge_in_the_last_24_hours - system_version: version_4_0 - hydrological_model: lisflood - leadtime_hour: '24' - metadata_mapping: - <<: *hday_hmonth_hyear - EFAS_FORECAST: - dataset: efas-forecast - format: grib - download_format: zip - system_version: operational - originating_centre: ecmwf - api_product_type: control_forecast - variable: river_discharge_in_the_last_6_hours - model_levels: surface_level - time: 00:00 - leadtime_hour: '24' - metadata_mapping: - <<: *day_month_year - EFAS_HISTORICAL: - dataset: efas-historical - format: grib - download_format: zip - model_levels: surface_level - variable: snow_depth_water_equivalent - system_version: version_5_0 - time: 00:00 - metadata_mapping: - <<: *hday_hmonth_hyear - EFAS_REFORECAST: - dataset: efas-reforecast - system_version: version_5_0 - format: grib - download_format: zip - api_product_type: control_forecast - variable: river_discharge_in_the_last_6_hours - model_levels: surface_level - leadtime_hour: '6' - metadata_mapping: - <<: *hday_hmonth_hyear - EFAS_SEASONAL: - dataset: efas-seasonal - system_version: operational - format: grib - download_format: zip - variable: river_discharge_in_the_last_24_hours - model_levels: surface_level - leadtime_hour: '24' - metadata_mapping: - <<: *month_year - EFAS_SEASONAL_REFORECAST: - dataset: efas-seasonal-reforecast - system_version: version_5_0 - format: grib - download_format: zip - variable: river_discharge_in_the_last_24_hours - model_levels: surface_level - leadtime_hour: '24' - metadata_mapping: - completionTimeFromAscendingNode: - - | - {{ - "hyear": {_date#interval_to_datetime_dict}["year"], - "hmonth": {_date#interval_to_datetime_dict}["month"] - }} - - '{$.completionTimeFromAscendingNode#to_iso_date}' SATELLITE_CARBON_DIOXIDE: dataset: satellite-carbon-dioxide processing_level: level_2 @@ -4630,16 +4472,6 @@ }} - $.properties.startDate completionTimeFromAscendingNode: $.properties.endDate - anchor_hist_day_month_year: &hist_day_month_year - startTimeFromAscendingNode: - - | - {{ - "hyear": {startTimeFromAscendingNode#to_datetime_dict(list)}["year"], - "hmonth": {startTimeFromAscendingNode#to_datetime_dict(list)}["month"], - "hday": {startTimeFromAscendingNode#to_datetime_dict(list)}["day"] - }} - - $.properties.startDate - completionTimeFromAscendingNode: $.properties.endDate anchor_year_month: &month_year startTimeFromAscendingNode: - | @@ -4649,15 +4481,6 @@ }} - $.properties.startDate completionTimeFromAscendingNode: $.properties.endDate - anchor_hist_year_month: &hist_month_year - startTimeFromAscendingNode: - - | - {{ - "hyear": {startTimeFromAscendingNode#to_datetime_dict(list)}["year"], - "hmonth": {startTimeFromAscendingNode#to_datetime_dict(list)}["month"] - }} - - $.properties.startDate - completionTimeFromAscendingNode: $.properties.endDate anchor_variable_list: &variable_list variable: - '{{"variable": {variable}}}' @@ -5578,324 +5401,26 @@ - '$.null' defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:ECMWF:DAT:CAMS_SOLAR_RADIATION_TIMESERIES"}}' - FIRE_HISTORICAL: - productType: EO:ECMWF:DAT:CEMS_FIRE_HISTORICAL_V1 - providerProductType: reanalysis - variable: - - fire_danger_index - version: - - "4_1" - format: - - "grib" - grid: original_grid - type: consolidated_dataset - metadata_mapping: - id: '$.id' - <<: *day_month_year - providerProductType: - - '{{"product_type": "{providerProductType}"}}' - - '$.null' - version: - - '{{"system_version": {version}}}' - - '$.null' - format: - - '{{"format": {format}}}' - - '$.null' - type: - - '{{"dataset_type": "{type}"}}' - - '$.null' - <<: *variable_list - defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' - orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:ECMWF:DAT:CEMS_FIRE_HISTORICAL_V1"}}' - GLOFAS_FORECAST: - productType: EO:ECMWF:DAT:CEMS_GLOFAS_FORECAST - variable: river_discharge_in_the_last_24_hours - version: - - operational - leadtime_hour: - - "24" - model: - - lisflood - providerProductType: - - control_forecast - format: grib - metadata_mapping: - id: '$.id' - <<: *day_month_year - providerProductType: - - '{{"product_type": {providerProductType}}}' - - '$.null' - version: - - '{{"system_version": {version}}}' - - '$.null' - model: - - '{{"hydrological_model": {model}}}' - - '$.null' - variable: - - '{{"variable": "{variable}"}}' - - '$.null' - defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' - orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:ECMWF:DAT:CEMS_GLOFAS_FORECAST"}}' - GLOFAS_HISTORICAL: - productType: EO:ECMWF:DAT:CEMS_GLOFAS_HISTORICAL - variable: - - river_discharge_in_the_last_24_hours - version: - - version_4_0 - model: - - lisflood - providerProductType: - - consolidated - format: grib - metadata_mapping: - id: '$.id' - <<: *hist_day_month_year - providerProductType: - - '{{"product_type": {providerProductType}}}' - - '$.null' - version: - - '{{"system_version": {version}}}' - - '$.null' - model: - - '{{"hydrological_model": {model}}}' - - '$.null' - <<: *variable_list - defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' - orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:ECMWF:DAT:CEMS_GLOFAS_HISTORICAL"}}' - GLOFAS_REFORECAST: - productType: EO:ECMWF:DAT:CEMS_GLOFAS_REFORECAST - variable: - - river_discharge_in_the_last_24_hours - providerProductType: - - control_reforecast - leadtime_hour: - - "24" - version: - - version_4_0 - model: - - lisflood - format: grib + auth: !plugin + type: TokenAuth + matching_url: https://[-\w\.]+.wekeo2.eu + auth_uri: 'https://gateway.prod.wekeo2.eu/hda-broker/gettoken' + refresh_uri: 'https://gateway.prod.wekeo2.eu/hda-broker/refreshtoken' + token_type: json + token_key: access_token + refresh_token_key: refresh_token + download: !plugin + type: HTTPDownload + auth_error_code: 401 + order_enabled: true + order_method: 'POST' + order_on_response: metadata_mapping: - id: '$.id' - <<: *hist_day_month_year - providerProductType: - - '{{"product_type": {providerProductType}}}' - - '$.null' - version: - - '{{"system_version": {version}}}' - - '$.null' - model: - - '{{"hydrological_model": {model}}}' - - '$.null' - <<: *variable_list - defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' - orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:ECMWF:DAT:CEMS_GLOFAS_REFORECAST"}}' - GLOFAS_SEASONAL: - productType: EO:ECMWF:DAT:CEMS_GLOFAS_SEASONAL - variable: - - river_discharge_in_the_last_24_hours - version: - - operational - model: - - htessel_lisflood - leadtime_hour: - - "24" - format: grib - metadata_mapping: - id: '$.id' - <<: *month_year - version: - - '{{"system_version": {version}}}' - - '$.null' - model: - - '{{"hydrological_model": {model}}}' - - '$.null' - <<: *variable_list - defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' - orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:ECMWF:DAT:CEMS_GLOFAS_SEASONAL"}}' - GLOFAS_SEASONAL_REFORECAST: - productType: EO:ECMWF:DAT:CEMS_GLOFAS_SEASONAL_REFORECAST - variable: - - river_discharge_in_the_last_24_hours - version: - - version_4_0 - model: - - lisflood - leadtime_hour: - - "24" - format: grib - metadata_mapping: - id: '$.id' - <<: *hist_month_year - version: - - '{{"system_version": {version}}}' - - '$.null' - model: - - '{{"hydrological_model": {model}}}' - - '$.null' - <<: *variable_list - defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' - orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:ECMWF:DAT:CEMS_GLOFAS_SEASONAL_REFORECAST"}}' - EFAS_FORECAST: - productType: EO:ECMWF:DAT:EFAS_FORECAST - providerProductType: - - control_forecast - soil_level: - - "1" - leadtime_hour: - - "0" - version: - - operational - origin: ecmwf - variable: volumetric_soil_moisture - model_level: soil_levels - time: - - 00:00 - format: grib.zip - metadata_mapping: - id: '$.id' - <<: *day_month_year - providerProductType: - - '{{"product_type": {providerProductType}}}' - - '$.null' - version: - - '{{"system_version": {version}}}' - - '$.null' - model_level: - - '{{"model_levels": "{model_level}"}}' - - '$.null' - origin: - - '{{"originating_centre": "{origin}"}}' - - '$.null' - variable: - - '{{"variable": "{variable}"}}' - - '$.null' - defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' - orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:ECMWF:DAT:EFAS_FORECAST"}}' - EFAS_HISTORICAL: - productType: EO:ECMWF:DAT:EFAS_HISTORICAL - version: - - version_5_0 - variable: volumetric_soil_moisture - model_level: soil_levels - soil_level: - - "1" - time: - - 00:00 - metadata_mapping: - id: '$.id' - <<: *hist_day_month_year - version: - - '{{"system_version": {version}}}' - - '$.null' - model_level: - - '{{"model_levels": "{model_level}"}}' - - '$.null' - variable: - - '{{"variable": "{variable}"}}' - - '$.null' - defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' - orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:ECMWF:DAT:EFAS_HISTORICAL"}}' - EFAS_REFORECAST: - productType: EO:ECMWF:DAT:EFAS_REFORECAST - providerProductType: - - control_forecast - version: - - version_5_0 - soil_level: - - "1" - leadtime_hour: - - "6" - variable: volumetric_soil_moisture - model_level: soil_levels - format: grib.zip - metadata_mapping: - id: '$.id' - <<: *hist_day_month_year - providerProductType: - - '{{"product_type": {providerProductType}}}' - - '$.null' - version: - - '{{"system_version": {version}}}' - - '$.null' - model_level: - - '{{"model_levels": "{model_level}"}}' - - '$.null' - variable: - - '{{"variable": "{variable}"}}' - - '$.null' - defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' - orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:ECMWF:DAT:EFAS_REFORECAST"}}' - EFAS_SEASONAL: - productType: EO:ECMWF:DAT:EFAS_SEASONAL - version: - - operational - leadtime_hour: - - "24" - variable: volumetric_soil_moisture - model_level: soil_levels - soil_level: - - "1" - format: grib.zip - metadata_mapping: - id: '$.id' - <<: *month_year - version: - - '{{"system_version": {version}}}' - - '$.null' - model_level: - - '{{"model_levels": "{model_level}"}}' - - '$.null' - variable: - - '{{"variable": "{variable}"}}' - - '$.null' - defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' - orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:ECMWF:DAT:EFAS_SEASONAL"}}' - EFAS_SEASONAL_REFORECAST: - productType: EO:ECMWF:DAT:EFAS_SEASONAL_REFORECAST - version: - - "version_5_0" - leadtime_hour: - - "24" - variable: volumetric_soil_moisture - model_level: soil_levels - soil_level: - - "1" - format: grib.zip - metadata_mapping: - id: '$.id' - <<: *hist_month_year - version: - - '{{"system_version": {version}}}' - - '$.null' - model_level: - - '{{"model_levels": "{model_level}"}}' - - '$.null' - variable: - - '{{"variable": "{variable}"}}' - - '$.null' - defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' - orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:ECMWF:DAT:EFAS_SEASONAL_REFORECAST"}}' - auth: !plugin - type: TokenAuth - matching_url: https://[-\w\.]+.wekeo2.eu - auth_uri: 'https://gateway.prod.wekeo2.eu/hda-broker/gettoken' - refresh_uri: 'https://gateway.prod.wekeo2.eu/hda-broker/refreshtoken' - token_type: json - token_key: access_token - refresh_token_key: refresh_token - download: !plugin - type: HTTPDownload - auth_error_code: 401 - order_enabled: true - order_method: 'POST' - order_on_response: - metadata_mapping: - orderId: '$.json.download_id' - orderStatusLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/downloads?download_id={orderId}' - order_status: - request: - method: GET + orderId: '$.json.download_id' + orderStatusLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/downloads?download_id={orderId}' + order_status: + request: + method: GET metadata_mapping: status: $.json.features[0].status message: $.json.features[0].message @@ -5909,19 +5434,6 @@ metadata_mapping: download_id: $.json.features[0]._id downloadLink: https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download/{download_id} - products: - GLOFAS_FORECAST: - output_extension: .grib - GLOFAS_REFORECAST: - output_extension: .grib - GLOFAS_HISTORICAL: - output_extension: .grib - GLOFAS_SEASONAL: - output_extension: .grib - GLOFAS_SEASONAL_REFORECAST: - output_extension: .grib - FIRE_HISTORICAL: - output_extension: .grib --- !provider # MARK: wekeo_cmems @@ -7450,7 +6962,7 @@ s3_endpoint: https://s3.waw3-1.cloudferro.com --- -!provider # MARK: geodes +!provider # MARK: geodes name: geodes priority: 0 roles: @@ -7575,3 +7087,299 @@ matching_url: https://geodes-portal.cnes.fr headers: X-API-Key: "{apikey}" + +--- +!provider # MARK: cop_ewds + name: cop_ewds + priority: 0 + description: CEMS Early Warning Data Store + roles: + - host + url: https://ewds.climate.copernicus.eu + anchor_day_month_year: &day_month_year + completionTimeFromAscendingNode: + - | + {{ + "year": {_date#interval_to_datetime_dict}["year"], + "month": {_date#interval_to_datetime_dict}["month"], + "day": {_date#interval_to_datetime_dict}["day"] + }} + - '{$.completionTimeFromAscendingNode#to_iso_date}' + anchor_month_year: &month_year + completionTimeFromAscendingNode: + - | + {{ + "year": {_date#interval_to_datetime_dict}["year"], + "month": {_date#interval_to_datetime_dict}["month"] + }} + - '{$.completionTimeFromAscendingNode#to_iso_date}' + anchor_hday_hmonth_hyear: &hday_hmonth_hyear + completionTimeFromAscendingNode: + - | + {{ + "hyear": {_date#interval_to_datetime_dict}["year"], + "hmonth": {_date#interval_to_datetime_dict}["month"], + "hday": {_date#interval_to_datetime_dict}["day"] + }} + - '{$.completionTimeFromAscendingNode#to_iso_date}' + anchor_hmonth_hyear: &hmonth_hyear + completionTimeFromAscendingNode: + - | + {{ + "hyear": {_date#interval_to_datetime_dict}["year"], + "hmonth": {_date#interval_to_datetime_dict}["month"] + }} + - '{$.completionTimeFromAscendingNode#to_iso_date}' + auth: !plugin + type: HTTPHeaderAuth + matching_url: https://ewds.climate.copernicus.eu + ssl_verify: true + headers: + PRIVATE-TOKEN: "{apikey}" + download: !plugin + type: HTTPDownload + timeout: 30 + ssl_verify: true + extract: false + auth_error_code: 401 + order_enabled: true + order_method: POST + order_on_response: + metadata_mapping: + orderId: $.json.jobID + orderStatusLink: https://ewds.climate.copernicus.eu/api/retrieve/v1/jobs/{orderId} + searchLink: https://ewds.climate.copernicus.eu/api/retrieve/v1/jobs/{orderId}/results + order_status: + request: + method: GET + metadata_mapping: + status: $.json.status + error: + status: failed + success: + status: successful + on_success: + need_search: true + metadata_mapping: + downloadLink: $.json.asset.value.href + products: + FIRE_HISTORICAL: + output_extension: .grib + search: !plugin + type: BuildSearchResult + ssl_verify: true + end_date_excluded: false + remove_from_query: + - dataset + - date + discover_queryables: + fetch_url: null + product_type_fetch_url: null + constraints_file_url: https://ewds.climate.copernicus.eu/api/catalogue/v1/collections/{dataset}/constraints.json + metadata_mapping: + productType: $.productType + title: $.id + startTimeFromAscendingNode: '{$.startTimeFromAscendingNode#to_iso_utc_datetime}' + completionTimeFromAscendingNode: + - date={startTimeFromAscendingNode#to_iso_date}/{completionTimeFromAscendingNode#to_iso_date} + - '{$.completionTimeFromAscendingNode#to_iso_utc_datetime}' + _date: '{startTimeFromAscendingNode#to_iso_date}/{completionTimeFromAscendingNode#to_iso_date}' + id: $.id + # The geographic extent of the product + geometry: + - '{{"area": {geometry#to_nwse_bounds}}}' + - $.geometry + defaultGeometry: POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90)) + storageStatus: '{$.null#replace_str("Not Available","OFFLINE")}' + downloadLink: $.null + qs: $.qs + orderLink: 'https://ewds.climate.copernicus.eu/api/retrieve/v1/processes/{dataset}/execution?{{"inputs": {qs#to_geojson}}}' + # Copernicus CDS specific parameters + api_product_type: + - product_type + - $.api_product_type + dataset: + - dataset + - $.dataset + dataset_type: + - dataset_type + - $.dataset_type + day: + - day + - $.day + download_format: + - download_format + - $.download_format + format: + - data_format + - $.format + grid: + - grid + - $.grid + hday: + - hday + - $.hday + hmonth: + - hmonth + - $.hmonth + hyear: + - hyear + - $.hyear + hydrological_model: + - hydrological_model + - $.hydrological_model + leadtime_hour: + - leadtime_hour + - $.leadtime_hour + model_levels: + - model_levels + - $.model_levels + month: + - month + - $.month + originating_centre: + - originating_centre + - $.originating_centre + soil_level: + - soil_level + - $.soil_level + system_version: + - system_version + - '{$.system_version#to_geojson}' + variable: + - variable + - $.variable + time: + - time + - $.time + year: + - year + - $.year + products: + EFAS_HISTORICAL: + dataset: efas-historical + format: grib + download_format: zip + model_levels: surface_level + variable: snow_depth_water_equivalent + system_version: version_5_0 + time: 00:00 + metadata_mapping: + <<: *hday_hmonth_hyear + EFAS_FORECAST: + dataset: efas-forecast + format: grib + download_format: zip + system_version: operational + originating_centre: ecmwf + api_product_type: control_forecast + variable: river_discharge_in_the_last_6_hours + model_levels: surface_level + time: 00:00 + leadtime_hour: '24' + metadata_mapping: + <<: *day_month_year + EFAS_SEASONAL: + dataset: efas-seasonal + system_version: operational + format: grib + download_format: zip + variable: river_discharge_in_the_last_24_hours + model_levels: surface_level + leadtime_hour: '24' + metadata_mapping: + <<: *month_year + EFAS_REFORECAST: + dataset: efas-reforecast + system_version: version_5_0 + format: grib + download_format: zip + api_product_type: control_forecast + variable: river_discharge_in_the_last_6_hours + model_levels: surface_level + leadtime_hour: '6' + metadata_mapping: + <<: *hday_hmonth_hyear + EFAS_SEASONAL_REFORECAST: + dataset: efas-seasonal-reforecast + system_version: version_5_0 + format: grib + download_format: zip + variable: river_discharge_in_the_last_24_hours + model_levels: surface_level + leadtime_hour: '24' + metadata_mapping: + <<: *hmonth_hyear + GLOFAS_HISTORICAL: + dataset: cems-glofas-historical + system_version: version_4_0 + variable: river_discharge_in_the_last_24_hours + format: grib2 + download_format: zip + hydrological_model: lisflood + api_product_type: consolidated + metadata_mapping: + <<: *hday_hmonth_hyear + GLOFAS_FORECAST: + dataset: cems-glofas-forecast + system_version: operational + variable: river_discharge_in_the_last_24_hours + format: grib2 + hydrological_model: htessel_lisflood + api_product_type: control_forecast + leadtime_hour: '24' + download_format: zip + metadata_mapping: + <<: *day_month_year + GLOFAS_SEASONAL: + dataset: cems-glofas-seasonal + variable: river_discharge_in_the_last_24_hours + format: grib2 + download_format: zip + system_version: operational + hydrological_model: lisflood + leadtime_hour: '24' + metadata_mapping: + <<: *month_year + GLOFAS_SEASONAL_REFORECAST: + dataset: cems-glofas-seasonal-reforecast + format: grib2 + download_format: zip + variable: river_discharge_in_the_last_24_hours + system_version: version_4_0 + hydrological_model: lisflood + leadtime_hour: '24' + metadata_mapping: + <<: *hmonth_hyear + GLOFAS_REFORECAST: + dataset: cems-glofas-reforecast + variable: river_discharge_in_the_last_24_hours + format: grib2 + download_format: zip + system_version: version_4_0 + hydrological_model: lisflood + api_product_type: control_reforecast + leadtime_hour: '24' + metadata_mapping: + <<: *hday_hmonth_hyear + FIRE_HISTORICAL: + dataset: cems-fire-historical-v1 + grid: original_grid + dataset_type: consolidated_dataset + api_product_type: reanalysis + variable: build_up_index + system_version: '4_1' + format: grib + metadata_mapping: + <<: *day_month_year + FIRE_SEASONAL: + dataset: cems-fire-seasonal + day: '01' + leadtime_hour: '12' + variable: build_up_index + system_version: '5' + metadata_mapping: + system_version: + - release_version + - $.system_version + <<: *month_year diff --git a/eodag/resources/user_conf_template.yml b/eodag/resources/user_conf_template.yml index e9729747c..9d319c1ce 100644 --- a/eodag/resources/user_conf_template.yml +++ b/eodag/resources/user_conf_template.yml @@ -66,6 +66,13 @@ cop_dataspace: credentials: username: password: +cop_ewds: + priority: # Lower value means lower priority (Default: 0) + download: + output_dir: + auth: + credentials: + apikey: cop_marine: priority: # Lower value means lower priority (Default: 0) search: # Search parameters configuration diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index f17ab7c4a..fb483bda7 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -194,6 +194,13 @@ # request for grib file instead of netcdf "format": "grib", } +COP_EWDS_SEARCH_ARGS = [ + "cop_ewds", + "EFAS_SEASONAL", + "2021-01-01", + "2021-01-05", + [-180, -90, 180, 90], +] SARA_SEARCH_ARGS = [ "sara", "S2_MSI_L1C", @@ -534,6 +541,11 @@ def test_end_to_end_search_download_cop_cds(self): expected_filename = "{}.grib".format(product.properties["title"]) self.execute_download(product, expected_filename) + def test_end_to_end_search_download_cop_ewds(self): + product = self.execute_search(*COP_EWDS_SEARCH_ARGS) + expected_filename = "{}.zip".format(product.properties["title"]) + self.execute_download(product, expected_filename) + def test_end_to_end_search_download_sara(self): product = self.execute_search(*SARA_SEARCH_ARGS) expected_filename = "{}.zip".format(product.properties["title"]) diff --git a/tests/units/test_core.py b/tests/units/test_core.py index fcc5e8735..64ba254b9 100644 --- a/tests/units/test_core.py +++ b/tests/units/test_core.py @@ -140,24 +140,25 @@ class TestCore(TestCoreBase): "DT_EXTREMES": ["dedl", "dedt_lumi"], "DT_CLIMATE_ADAPTATION": ["dedl", "dedt_lumi"], "EEA_DAILY_VI": ["dedl", "wekeo_main"], - "EFAS_FORECAST": ["cop_cds", "dedl", "wekeo_ecmwf"], - "EFAS_HISTORICAL": ["cop_cds", "dedl", "wekeo_ecmwf"], - "EFAS_REFORECAST": ["cop_cds", "dedl", "wekeo_ecmwf"], - "EFAS_SEASONAL": ["cop_cds", "dedl", "wekeo_ecmwf"], - "EFAS_SEASONAL_REFORECAST": ["cop_cds", "dedl", "wekeo_ecmwf"], + "EFAS_FORECAST": ["cop_ewds", "dedl"], + "EFAS_HISTORICAL": ["cop_ewds", "dedl"], + "EFAS_REFORECAST": ["cop_ewds", "dedl"], + "EFAS_SEASONAL": ["cop_ewds", "dedl"], + "EFAS_SEASONAL_REFORECAST": ["cop_ewds", "dedl"], "ERA5_LAND": ["cop_cds", "dedl", "wekeo_ecmwf"], "ERA5_LAND_MONTHLY": ["cop_cds", "dedl", "wekeo_ecmwf"], "ERA5_PL": ["cop_cds", "dedl", "wekeo_ecmwf"], "ERA5_PL_MONTHLY": ["cop_cds", "dedl", "wekeo_ecmwf"], "ERA5_SL": ["cop_cds", "dedl", "wekeo_ecmwf"], "ERA5_SL_MONTHLY": ["cop_cds", "dedl", "wekeo_ecmwf"], - "FIRE_HISTORICAL": ["cop_cds", "dedl", "wekeo_ecmwf"], + "FIRE_HISTORICAL": ["cop_ewds", "dedl"], + "FIRE_SEASONAL": ["cop_ewds"], "GLACIERS_DIST_RANDOLPH": ["cop_cds", "dedl", "wekeo_ecmwf"], - "GLOFAS_FORECAST": ["cop_cds", "dedl", "wekeo_ecmwf"], - "GLOFAS_HISTORICAL": ["cop_cds", "dedl", "wekeo_ecmwf"], - "GLOFAS_REFORECAST": ["cop_cds", "dedl", "wekeo_ecmwf"], - "GLOFAS_SEASONAL": ["cop_cds", "dedl", "wekeo_ecmwf"], - "GLOFAS_SEASONAL_REFORECAST": ["cop_cds", "dedl", "wekeo_ecmwf"], + "GLOFAS_FORECAST": ["cop_ewds", "dedl"], + "GLOFAS_HISTORICAL": ["cop_ewds", "dedl"], + "GLOFAS_REFORECAST": ["cop_ewds", "dedl"], + "GLOFAS_SEASONAL": ["cop_ewds", "dedl"], + "GLOFAS_SEASONAL_REFORECAST": ["cop_ewds", "dedl"], "GRIDDED_GLACIERS_MASS_CHANGE": ["cop_cds", "dedl", "wekeo_ecmwf"], "L57_REFLECTANCE": ["theia"], "L8_OLI_TIRS_C1L1": ["aws_eos", "earth_search_gcs", "onda"], @@ -597,6 +598,7 @@ class TestCore(TestCoreBase): "cop_ads", "cop_cds", "cop_dataspace", + "cop_ewds", "cop_marine", "creodias", "creodias_s3", @@ -1535,6 +1537,7 @@ def test_available_sortables(self): ], "max_sort_params": 1, }, + "cop_ewds": None, "creodias": { "sortables": [ "startTimeFromAscendingNode",