From 25aafc0e71a1baa3cb212cc31cf6bb2089d132dc Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Thu, 8 Aug 2024 10:27:16 +0200 Subject: [PATCH 01/25] docs: config parameters for auth plugins --- eodag/plugins/authentication/aws_auth.py | 18 ++- eodag/plugins/authentication/generic.py | 11 +- eodag/plugins/authentication/header.py | 7 +- eodag/plugins/authentication/keycloak.py | 14 +- eodag/plugins/authentication/oauth.py | 9 +- .../plugins/authentication/openid_connect.py | 124 +++++++----------- eodag/plugins/authentication/qsauth.py | 5 + eodag/plugins/authentication/sas_auth.py | 13 +- eodag/plugins/authentication/token.py | 17 ++- .../plugins/authentication/token_exchange.py | 26 ++-- 10 files changed, 139 insertions(+), 105 deletions(-) diff --git a/eodag/plugins/authentication/aws_auth.py b/eodag/plugins/authentication/aws_auth.py index 4ca0f9c72..a10a646a6 100644 --- a/eodag/plugins/authentication/aws_auth.py +++ b/eodag/plugins/authentication/aws_auth.py @@ -30,14 +30,22 @@ class AwsAuth(Authentication): """AWS authentication plugin - Authentication will use the first valid method within the following ones: + Authentication will use the first valid method within the following ones depending on which + parameters are available in the configuration: - - auth anonymously using no-sign-request - - auth using ``aws_profile`` - - auth using ``aws_access_key_id`` and ``aws_secret_access_key`` + * auth anonymously using no-sign-request + * auth using ``aws_profile`` + * auth using ``aws_access_key_id`` and ``aws_secret_access_key`` (optionally ``aws_session_token``) - - auth using current environment (AWS environment variables and/or ``~/aws/*``), + * auth using current environment (AWS environment variables and/or ``~/aws/*``), will be skipped if AWS credentials are filled in eodag conf + + The configuration parameters for this plugin are: + + * **type** [str] (mandatory): AwsAuth + * **auth_error_code** [int] (mandatory for creodias_s3): which error code is returned + in case of an authentication error + """ s3_client: S3Client diff --git a/eodag/plugins/authentication/generic.py b/eodag/plugins/authentication/generic.py index 9ec482280..b1da2f912 100644 --- a/eodag/plugins/authentication/generic.py +++ b/eodag/plugins/authentication/generic.py @@ -29,7 +29,16 @@ class GenericAuth(Authentication): - """GenericAuth authentication plugin""" + """GenericAuth authentication plugin (authentication using username and password) + + The configuration parameters for this plugin are: + + * **type** [str] (mandatory): GenericAuth + * **method** [str]: specifies if digest authentication (digest) or basic authentication + (basic) should be used; default: basic + + The mandatory parameters that have to be added in the eodag config are username and password. + """ def authenticate(self) -> AuthBase: """Authenticate""" diff --git a/eodag/plugins/authentication/header.py b/eodag/plugins/authentication/header.py index 2fed1502b..60bcf77eb 100644 --- a/eodag/plugins/authentication/header.py +++ b/eodag/plugins/authentication/header.py @@ -34,7 +34,12 @@ class HTTPHeaderAuth(Authentication): This plugin enables implementation of custom HTTP authentication scheme (other than Basic, Digest, Token negotiation et al.) using HTTP headers. - The plugin is configured as follows in the providers config file:: + The configuration parameters for this plugin are: + + * **type** [str] (mandatory): HTTPHeaderAuth + * **headers** [Dict[str, str]]: dictionary containing all keys/value pairs that should be added to the headers + + Below an example for the configuration in the providers config file is shown:: provider: ... diff --git a/eodag/plugins/authentication/keycloak.py b/eodag/plugins/authentication/keycloak.py index e0a7ebda8..ab7d4b448 100644 --- a/eodag/plugins/authentication/keycloak.py +++ b/eodag/plugins/authentication/keycloak.py @@ -41,7 +41,19 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase): """Authentication plugin using Keycloak and OpenId Connect. - This plugin request a token and use it through a query-string or a header. + This plugin requests a token which is added to a query-string or a header for authentication. + The configuration parameters for this plugin are: + + * **type** [str] (mandatory): KeycloakOIDCPasswordAuth + * **auth_base_uri** [str] (mandatory): base url used in the request to fetch the token + * **realm** [str] (mandatory): keycloak realm + * **client_id** [str] (mandatory): keycloak client id + * **client_secret** [str] (mandatory): keycloak client secret, set to null if no secret is used + * **token_provision** [str] (mandatory): if the token should be added to the query string (qs) + or to the header (header) + * **token_qs_key** [str] (mandatory if token_provision=qs): key of the param added to the query string + * **auth_error_code** [int]: which error code is returned in case of an authentication error + * **ssl_verify** [bool]: if the ssl certificates should be verified in the token request; default: True Using :class:`~eodag.plugins.download.http.HTTPDownload` a download link `http://example.com?foo=bar` will become diff --git a/eodag/plugins/authentication/oauth.py b/eodag/plugins/authentication/oauth.py index 8f31a3e5b..24ddea304 100644 --- a/eodag/plugins/authentication/oauth.py +++ b/eodag/plugins/authentication/oauth.py @@ -26,7 +26,14 @@ class OAuth(Authentication): - """OAuth authentication plugin""" + """OAuth authentication plugin + + The configuration parameters for this plugin are: + + * **type** [str] (mandatory): OAuth + + The mandatory parameters that have to be added in the eodag config are aws_access_key_id and aws_secret_access_key. + """ def __init__(self, provider: str, config: PluginConfig) -> None: super(OAuth, self).__init__(provider, config) diff --git a/eodag/plugins/authentication/openid_connect.py b/eodag/plugins/authentication/openid_connect.py index 2c0cc1da9..5bb782bfc 100644 --- a/eodag/plugins/authentication/openid_connect.py +++ b/eodag/plugins/authentication/openid_connect.py @@ -180,83 +180,53 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase): The headless interaction is fully configurable, and rely on XPATH to retrieve all the necessary information. - The configuration keys of this plugin are as follows (they have no defaults):: - - # (mandatory) The authorization url of the server (where to query for grants) - authorization_uri: - - # (mandatory) The callback url that will handle the code given by the OIDC provider - redirect_uri: - - # (mandatory) The url to query to exchange the authorization code obtained from the OIDC provider - # for an authorized token - token_uri: - - # (mandatory) The OIDC provider's client ID of the eodag provider - client_id: - - # (mandatory) Wether a user consent is needed during the authentication - user_consent_needed: - - # (mandatory) One of: json, data or params. This is the way to pass the data to the POST request - # that is made to the token server. They correspond to the recognised keywords arguments - # of the Python `requests `_ library - token_exchange_post_data_method: - - # (mandatory) The key pointing to the token in the json response to the POST request to the token server - token_key: - - # (mandatory) One of qs or header. This is how the token obtained will be used to authenticate the user - # on protected requests. If 'qs' is chosen, then 'token_qs_key' is mandatory - token_provision: - - # (mandatory) The xpath to the HTML form element representing the user login form - login_form_xpath: - - # (mandatory) Where to look for the authentication_uri. One of 'config' (in the configuration) or 'login-form' - # (use the 'action' URL found in the login form retrieved with login_form_xpath). If the value is 'config', - # authentication_uri config param is mandatory - authentication_uri_source: - - # (optional) The URL of the authentication backend of the OIDC provider - authentication_uri: - - # (optional) The xpath to the user consent form. The form is searched in the content of the response - # to the authorization request - user_consent_form_xpath: - - # (optional) The data that will be passed with the POST request on the form 'action' URL. The data are - # given as a key value pairs, the keys representing the data key and the value being either - # a 'constant' string value, or a string of the form 'xpath()' - # and representing a value to be retrieved in the user consent form. The xpath must resolve - # directly to a string value, not to an HTML element. Example: - # `xpath(//input[@name="sessionDataKeyConsent"]/@value)` - user_consent_form_data: - - # (optional) A mapping giving additional data to be passed to the login POST request. The value follows the - # same rules as with user_consent_form_data - additional_login_form_data: - - # (optional) Key/value pairs of patterns/messages. If exchange_url contains the given pattern, the associated - message will be sent in an AuthenticationError - exchange_url_error_pattern: - - # (optional) The OIDC provider's client secret of the eodag provider - client_secret: - - # (optional) A mapping between OIDC url query string and token handler query string - # params (only necessary if they are not the same as for OIDC). This is eodag provider - # dependant - token_exchange_params: - redirect_uri: - client_id: - - # (optional) Only necessary when 'token_provision' is 'qs'. Refers to the name of the query param to be - # used in the query request - token_qs_key: - - # (optional) The key pointing to the refresh_token in the json response to the POST request to the token server - refresh_token_key: + The configuration keys of this plugin are as follows (they have no defaults): + + * **type** [str] (mandatory): OIDCAuthorizationCodeFlowAuth + * **authorization_uri** [str] (mandatory): The authorization url of the server (where to query for grants) + * **redirect_uri** [str] (mandatory): The callback url that will handle the code given by the OIDC provider + * **token_uri** [str] (mandatory): The url to query to exchange the authorization + code obtained from the OIDC providerfor an authorized token + * **client_id** [str] (mandatory): The OIDC provider's client ID of the eodag provider + * **user_consent_needed** [bool] (mandatory): Whether a user consent is needed + during the authentication + * **token_exchange_post_data_method** [str] (mandatory): One of: json, data or params. + This is the way to pass the data to the POST request that is made to the token server. + They correspond to the recognised keywords arguments of the Python + `requests `_ library + * **token_key** [str] (mandatory): The key pointing to the token in the json response + to the POST request to the token server + * **token_provision** [str] (mandatory): One of qs or header. This is how the token + obtained will be used to authenticate the user on protected requests. If 'qs' is + chosen, then 'token_qs_key' is mandatory + * **login_form_xpath** [str] (mandatory): The xpath to the HTML form element representing + the user login form + * **authentication_uri_source** [str] (mandatory): Where to look for the authentication_uri. + One of 'config' (in the configuration) or 'login-form' (use the 'action' URL found in the + login form retrieved with login_form_xpath). If the value is 'config', + authentication_uri config param is mandatory + * **authentication_uri** [str] (mandatory if authentication_uri_source=config): + The URL of the authentication backend of the OIDC provider + * **user_consent_form_xpath** [str]: The xpath to the user consent form. The form + is searched in the content of the response to the authorization request + * **user_consent_form_data** [Dict[str, str]]: The data that will be passed with the + POST request on the form 'action' URL. The data are given as a key value pairs, the + keys representing the data key and the value being either a 'constant' string value, + or a string of the form 'xpath()' and representing a + value to be retrieved in the user consent form. The xpath must resolve directly to a + string value, not to an HTML element. Example: `xpath(//input[@name="sessionDataKeyConsent"]/@value)` + * **additional_login_form_data** [Dict[str, str]]: A mapping giving additional data + to be passed to the login POST request. The value follows the same rules as with user_consent_form_data + * **exchange_url_error_pattern** [Dict[str, str]]: Key/value pairs of patterns/messages. + If exchange_url contains the given pattern, the associated message will be sent in an AuthenticationError + * **client_secret** [str]:The OIDC provider's client secret of the eodag provider + * **token_exchange_params** [Dict[str, str]: mandatory keys for the dict: redirect_uri, client_id; + A mapping between OIDC url query string and token handler query string params + (only necessary if they are not the same as for OIDC). This is eodag provider dependant + * **token_qs_key** [str] (mandatory when token_provision=qs): Refers to the name + of the query param to be used in the query request + * **refresh_token_key** [str]: The key pointing to the refresh_token in the json response + to the POST request to the token server """ SCOPE = "openid" diff --git a/eodag/plugins/authentication/qsauth.py b/eodag/plugins/authentication/qsauth.py index f85f6938f..6e532db8b 100644 --- a/eodag/plugins/authentication/qsauth.py +++ b/eodag/plugins/authentication/qsauth.py @@ -36,6 +36,11 @@ class HttpQueryStringAuth(Authentication): """An Authentication plugin using HTTP query string parameters. This plugin sends credentials as query-string parameters. + The configuration parameters for this plugin are: + + * **type** [str] (mandatory): HttpQueryStringAuth + * **auth_uri** [str]: used to check the credentials given in the configuration + Using :class:`~eodag.plugins.download.http.HTTPDownload` a download link `http://example.com?foo=bar` will become `http://example.com?foo=bar&apikey=XXX&otherkey=YYY` if associated to the following diff --git a/eodag/plugins/authentication/sas_auth.py b/eodag/plugins/authentication/sas_auth.py index 3965a2632..e6040d238 100644 --- a/eodag/plugins/authentication/sas_auth.py +++ b/eodag/plugins/authentication/sas_auth.py @@ -85,7 +85,18 @@ def __call__(self, request: PreparedRequest) -> PreparedRequest: class SASAuth(Authentication): - """SASAuth authentication plugin""" + """SASAuth authentication plugin + + The configuration parameters for this plugin are: + + * **type** [str] (mandatory): SASAuth + * **auth_uri** [str] (mandatory): url used to get the signed url + * **signed_url_key** [str] (mandatory): key to get the signed url + * **headers** [Dict[str, str]] (mandatory if apiKey is used): headers to be added to the requests + * **ssl_verify** [bool]: if the ssl certificates should be verified in the requests; default: True + + An apiKey that is added in the headers can be given in the credentials in the config file. + """ def validate_config_credentials(self) -> None: """Validate configured credentials""" diff --git a/eodag/plugins/authentication/token.py b/eodag/plugins/authentication/token.py index 6cd89214b..55cc8e575 100644 --- a/eodag/plugins/authentication/token.py +++ b/eodag/plugins/authentication/token.py @@ -41,7 +41,22 @@ class TokenAuth(Authentication): - """TokenAuth authentication plugin""" + """TokenAuth authentication plugin - fetches a token which is added to search/download requests + + The configuration parameters for this plugin are: + + * **type** [str] (mandatory): TokenAuth + * **auth_uri** [str] (mandatory): url used to fetch the access token with user/password + * **refresh_uri** [str] : url used to fetch the access token with a refresh token + * **token_type** [str]: type of the token (json or text); default: text + * **token_key** [str] (mandatory if token_type=json): key to get the access token in the + response to the token request + * **refresh_token_key** [str]: key to get the refresh token in the response to the token request + * **ssl_verify** [bool]: if the ssl certificates should be verified in the requests; default: True + * **auth_error_code** [int]: which error code is returned in case of an authentication error + * **req_data** [Dict[str, Any]]: if the credentials should be sent as data in the post request, + the json structure can be given in this parameter + """ def __init__(self, provider: str, config: PluginConfig) -> None: super(TokenAuth, self).__init__(provider, config) diff --git a/eodag/plugins/authentication/token_exchange.py b/eodag/plugins/authentication/token_exchange.py index 36789e4ae..52dc24b68 100644 --- a/eodag/plugins/authentication/token_exchange.py +++ b/eodag/plugins/authentication/token_exchange.py @@ -38,25 +38,17 @@ class OIDCTokenExchangeAuth(Authentication): """Token exchange implementation using :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` token as subject. - The configuration keys of this plugin are as follows (they have no defaults):: + The configuration keys of this plugin are as follows (they have no defaults): - # (mandatory) The full OIDCAuthorizationCodeFlowAuth plugin configuration used to retrieve subject token - subject: + * **subject** [Dict[str, Any]] (mandatory): The full OIDCAuthorizationCodeFlowAuth plugin + configuration used to retrieve subject token + * **subject_issuer** [str] (mandatory): Identifies the issuer of the subject_token + * **token_uri** [str] (mandatory): The url to query to get the authorized token + * **client_id** [str] (mandatory): The OIDC provider's client ID of the eodag provider + * **audience** [str] (mandatory): This parameter specifies the target client you want the new token minted for. + * **token_key** [str] (mandatory): The key pointing to the token in the json response to + the POST request to the token server - # (mandatory) Identifies the issuer of the subject_token - subject_issuer: - - # (mandatory) The url to query to get the authorized token - token_uri: - - # (mandatory) The OIDC provider's client ID of the eodag provider - client_id: - - # (mandatory) This parameter specifies the target client you want the new token minted for. - audience: - - # (mandatory) The key pointing to the token in the json response to the POST request to the token server - token_key: """ GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange" From 81d5640782e0b84b7a1630b40fa82e2d4c22571b Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Thu, 8 Aug 2024 14:55:53 +0200 Subject: [PATCH 02/25] docs: config parameters for api plugins --- eodag/plugins/apis/ecmwf.py | 17 +++++++++++++---- eodag/plugins/apis/usgs.py | 16 +++++++++++++++- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/eodag/plugins/apis/ecmwf.py b/eodag/plugins/apis/ecmwf.py index 7ac0ea212..76d1d1567 100644 --- a/eodag/plugins/apis/ecmwf.py +++ b/eodag/plugins/apis/ecmwf.py @@ -68,10 +68,19 @@ class EcmwfApi(Api, BuildPostSearchResult): is in query), or on MARS Operational Archive (if ``dataset`` parameter is not in query). - This class inherits from :class:`~eodag.plugins.apis.base.Api` for compatibility, - :class:`~eodag.plugins.download.base.Download` for download methods, and - :class:`~eodag.plugins.search.qssearch.QueryStringSearch` for metadata-mapping and - query build methods. + This class inherits from :class:`~eodag.plugins.apis.base.Api` for compatibility and + :class:`~eodag.plugins.search.build_search_result.BuildPostSearchResult` for the creation + of the search result. + + The configuration parameters for this plugin are: + + * **type** [str] (mandatory): EcmwfApi + * **api_endpoint** [str] (mandatory): url of the ecmwf api + * **metadata_mapping** [Dict[str, Union[str, list]]]: how parameters should be mapped between + the provider and eodag; If a string is given, this is the mapping parameter returned by + provider -> eodag parameter. If a list with 2 elements is given, the first one is the mapping + eodag parameter -> provider query parameters and the second one the mapping provider result + parameter -> eodag parameter """ def __init__(self, provider: str, config: PluginConfig) -> None: diff --git a/eodag/plugins/apis/usgs.py b/eodag/plugins/apis/usgs.py index 4e7874107..5f78361f3 100644 --- a/eodag/plugins/apis/usgs.py +++ b/eodag/plugins/apis/usgs.py @@ -68,7 +68,21 @@ class UsgsApi(Api): - """A plugin that enables to query and download data on the USGS catalogues""" + """A plugin that enables to query and download data on the USGS catalogues + + The configuration parameters for this plugin are: + + * **type** [str] (mandatory): UsgsApi + * **pagination** [Dict[str, Any]] (mandatory): dict containing parameters for pagination; should contain the + key total_items_nb_key_path which is indicating the key for the number of total items in the provider result + * **ssl_verify** [bool]: if the ssl certificates should be verified in the download request; default: True + * **extract** [bool]: if the content of the downloaded file should be extracted; default: True + * **order_enabled** [bool]: if the product has to be ordered to download it; default: False + * **metadata_mapping** [Dict[str, Union[str, list]]]: how parameters should be mapped between the provider and eodag + If a string is given, this is the mapping parameter returned by provider -> eodag parameter. If a list with + 2 elements is given, the first one is the mapping eodag parameter -> provider query parameters and the second one + the mapping provider result parameter -> eodag parameter + """ def __init__(self, provider: str, config: PluginConfig) -> None: super(UsgsApi, self).__init__(provider, config) From cafc4186f68acacd767db24334433a414e200153 Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Fri, 9 Aug 2024 12:30:51 +0200 Subject: [PATCH 03/25] docs: config parameters for download plugins --- eodag/plugins/apis/usgs.py | 1 + eodag/plugins/download/aws.py | 28 ++++++++++++------ eodag/plugins/download/creodias_s3.py | 8 +++++- eodag/plugins/download/http.py | 41 +++++++++++++++++---------- 4 files changed, 53 insertions(+), 25 deletions(-) diff --git a/eodag/plugins/apis/usgs.py b/eodag/plugins/apis/usgs.py index 5f78361f3..d981ec017 100644 --- a/eodag/plugins/apis/usgs.py +++ b/eodag/plugins/apis/usgs.py @@ -76,6 +76,7 @@ class UsgsApi(Api): * **pagination** [Dict[str, Any]] (mandatory): dict containing parameters for pagination; should contain the key total_items_nb_key_path which is indicating the key for the number of total items in the provider result * **ssl_verify** [bool]: if the ssl certificates should be verified in the download request; default: True + * **need_auth** [bool]: if authentication is required for search; default: False * **extract** [bool]: if the content of the downloaded file should be extracted; default: True * **order_enabled** [bool]: if the product has to be ordered to download it; default: False * **metadata_mapping** [Dict[str, Union[str, list]]]: how parameters should be mapped between the provider and eodag diff --git a/eodag/plugins/download/aws.py b/eodag/plugins/download/aws.py index 1e35f210e..aeb9c6dd7 100644 --- a/eodag/plugins/download/aws.py +++ b/eodag/plugins/download/aws.py @@ -214,15 +214,25 @@ class AwsDownload(Download): """Download on AWS using S3 protocol. - :param provider: provider name - :param config: Download plugin configuration: - - * ``config.s3_endpoint`` (str) - s3 endpoint url - * ``config.requester_pays`` (bool) - (optional) whether download is done from a - requester-pays bucket or not - * ``config.flatten_top_dirs`` (bool) - (optional) flatten directory structure - * ``config.products`` (dict) - (optional) product_type specific configuration - * ``config.ignore_assets`` (bool) - (optional) ignore assets and download using downloadLink + The configuration parameters for this plugin are: + + * **type** [str] (mandatory): AwsDownload + * **base_uri** [str] (mandatory): s3 endpoint url + * **requester_pays** [bool]: whether download is done from a requester-pays bucket or not; default: False + * **flatten_top_dirs** [bool]: if the directory structure should be flattened; default: True + * **ignore_assets** [bool]: ignore assets and download using downloadLink; default: False + * **ssl_verify** [bool]: if the ssl certificates should be verified in requests; default: True + * **bucket_path_level** [int]: at which level of the path part of the url the bucket can be found; + If no bucket_path_level is given, the bucket is taken from the first element of the netloc part. + * **products** [Dict[str, Dict[str, Any]]: product type specific config; the keys are the product types, + the values are dictionaries which can contain the keys: + + * **default_bucket** [str]: bucket where the product type can be found + * **complementary_url_key** [str]: keys to add additional urls + * **build_safe** [bool]: if a SAFE (Standard Archive Format for Europe) product should + be created; used for Sentinel products; default: False + * **fetch_metadata** [Dict[str, Any]: config for metadata to be fetched for the SAFE product + """ def __init__(self, provider: str, config: PluginConfig) -> None: diff --git a/eodag/plugins/download/creodias_s3.py b/eodag/plugins/download/creodias_s3.py index 3c0f4052e..b4ad5cb61 100644 --- a/eodag/plugins/download/creodias_s3.py +++ b/eodag/plugins/download/creodias_s3.py @@ -25,7 +25,13 @@ class CreodiasS3Download(AwsDownload): """ - Download on creodias s3 from their VMs + Download on creodias s3 from their VMs (extension of AwsDownload) + The configuration parameters for this plugin are: + + * **type** [str] (mandatory): CreodiasS3Download + * **base_uri** [str] (mandatory): s3 endpoint url + * **s3_bucket** [str] (mandatory): bucket where the products can be found + * **ssl_verify** [bool]: if the ssl certificates should be verified in requests; default: True """ def _get_authenticated_objects_unsigned(self, bucket_name, prefix, auth_dict): diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py index 2302c53b9..756ae8955 100644 --- a/eodag/plugins/download/http.py +++ b/eodag/plugins/download/http.py @@ -98,21 +98,32 @@ class HTTPDownload(Download): """HTTPDownload plugin. Handles product download over HTTP protocol - :param provider: provider name - :param config: Download plugin configuration: - - * ``config.base_uri`` (str) - (optional) default endpoint url - * ``config.extract`` (bool) - (optional) extract downloaded archive or not - * ``config.auth_error_code`` (int) - (optional) authentication error code - * ``config.dl_url_params`` (dict) - (optional) attitional parameters to send in the request - * ``config.archive_depth`` (int) - (optional) level in extracted path tree where to find data - * ``config.flatten_top_dirs`` (bool) - (optional) flatten directory structure - * ``config.ignore_assets`` (bool) - (optional) ignore assets and download using downloadLink - * ``config.order_enabled`` (bool) - (optional) wether order is enabled or not if product is `OFFLINE` - * ``config.order_method`` (str) - (optional) HTTP request method, GET (default) or POST - * ``config.order_headers`` (dict) - (optional) order request headers - * ``config.order_on_response`` (dict) - (optional) edit or add new product properties - * ``config.order_status`` (:class:`~eodag.config.PluginConfig.OrderStatus`) - (optional) Order status handling + The configuration parameters for this plugin are: + + * **type** [str] (mandatory): HTTPDownload + * **base_uri** [str]: default endpoint url + * **method** [str]: HTTP request method for the download request (GET or POST); default: GET + * **extract** [bool]: if the content of the downloaded file should be extracted; default: True + * **auth_error_code** [int]: which error code is returned in case of an authentication error + * **dl_url_params** [Dict[str, Any]]: parameters to be added to the query params of the request + * **archive_depth** [int]: level in extracted path tree where to find data; default: 1 + * **flatten_top_dirs** [bool]: if the directory structure should be flattened; default: True + * **ignore_assets** [bool]: ignore assets and download using downloadLink; default: False + * **timeout** [int]: time to wait until request timeout in seconds; default: 5 + * **ssl_verify** [bool]: if the ssl certificates should be verified in requests; default: True + * **output_extension** [str]: which extension should be used for the downloaded file + * **no_auth_download** [bool]: if the download should be done without authentication; default: True + * **order_enabled** [bool]: if the product has to be ordered to download it; default: False + * **order_method** [str]: HTTP request method for the order request (GET or POST); default: GET + * **order_headers** [Dict[str, str]]: headers to be added to the order request + * **order_on_response** [Dict[str, Dict[str, Any]]: a dictionary containing the key 'metadata_mapping' + which can be used to add new product properties based on the data in response to the order request + * **order_status** [:class:`~eodag.config.PluginConfig.OrderStatus`]: configuration to handle + the order status; contains information which method to use, how the response data is interpreted, + which status corresponds to success, ordered and error and what should be done on success. + * **products** [Dict[str, Dict[str, Any]]: product type specific config; the keys are the product types, + the values are dictionaries which can contain the keys output_extension and extract to overwrite + the provider config for a specific product type """ From 59a07fb72b812c5c011e39c4769a43ff80140b4e Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Mon, 12 Aug 2024 18:17:18 +0200 Subject: [PATCH 04/25] docs: config params for QueryStringSearch --- eodag/plugins/authentication/aws_auth.py | 2 +- eodag/plugins/download/http.py | 5 +- eodag/plugins/search/qssearch.py | 208 ++++++++++++++--------- eodag/resources/providers.yml | 2 +- 4 files changed, 132 insertions(+), 85 deletions(-) diff --git a/eodag/plugins/authentication/aws_auth.py b/eodag/plugins/authentication/aws_auth.py index a10a646a6..02f52abf5 100644 --- a/eodag/plugins/authentication/aws_auth.py +++ b/eodag/plugins/authentication/aws_auth.py @@ -44,7 +44,7 @@ class AwsAuth(Authentication): * **type** [str] (mandatory): AwsAuth * **auth_error_code** [int] (mandatory for creodias_s3): which error code is returned - in case of an authentication error + in case of an authentication error """ diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py index 756ae8955..593d82846 100644 --- a/eodag/plugins/download/http.py +++ b/eodag/plugins/download/http.py @@ -116,8 +116,9 @@ class HTTPDownload(Download): * **order_enabled** [bool]: if the product has to be ordered to download it; default: False * **order_method** [str]: HTTP request method for the order request (GET or POST); default: GET * **order_headers** [Dict[str, str]]: headers to be added to the order request - * **order_on_response** [Dict[str, Dict[str, Any]]: a dictionary containing the key 'metadata_mapping' - which can be used to add new product properties based on the data in response to the order request + * **order_on_response** [:class:`~eodag.config.PluginConfig.OrderOnResponse`]: a typed dictionary + containing the key 'metadata_mapping' which can be used to add new product properties + based on the data in response to the order request * **order_status** [:class:`~eodag.config.PluginConfig.OrderStatus`]: configuration to handle the order status; contains information which method to use, how the response data is interpreted, which status corresponds to success, ordered and error and what should be done on success. diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index 892b01851..66e9317aa 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -110,98 +110,144 @@ class QueryStringSearch(Search): """A plugin that helps implementing any kind of search protocol that relies on - query strings (e.g: opensearch). + query strings (e.g: opensearch). Most of the other search plugins inherit from this plugin. The available configuration parameters for this kind of plugin are: - - **result_type**: (optional) One of "json" or "xml", depending on the - representation of the provider's search results. The default is "json" - - - **results_entry**: (mandatory) The name of the key in the provider search - result that gives access to the result entries - - - **api_endpoint**: (mandatory) The endpoint of the provider's search interface - - - **literal_search_params**: (optional) A mapping of (search_param => - search_value) pairs giving search parameters to be passed as is in the search - url query string. This is useful for example in situations where the user wants - to pass-in a search query as it is done on the provider interface. In such a case, - the user can put in his configuration file the query he needs to pass to the provider. - - - **pagination**: (mandatory) The configuration of how the pagination is done - on the provider. It is a tree with the following nodes: - - - *next_page_url_tpl*: The template for pagination requests. This is a simple - Python format string which will be resolved using the following keywords: - ``url`` (the base url of the search endpoint), ``search`` (the query string - corresponding to the search request), ``items_per_page`` (the number of - items to return per page), ``skip`` (the number of items to skip) or - ``skip_base_1`` (the number of items to skip, starting from 1) and - ``page`` (which page to return). - - - *total_items_nb_key_path*: (optional) An XPath or JsonPath leading to the - total number of results satisfying a request. This is used for providers - which provides the total results metadata along with the result of the - query and don't have an endpoint for querying the number of items - satisfying a request, or for providers for which the count endpoint returns - a json or xml document - - - *count_endpoint*: (optional) The endpoint for counting the number of items - satisfying a request - - - *next_page_url_key_path*: (optional) A JSONPATH expression used to retrieve + * **result_type** [str]: One of "json" or "xml", depending on the + representation of the provider's search results. The default is "json". + * **results_entry** [str] (mandatory): The name of the key in the provider search + result that gives access to the result entries + * **api_endpoint** [str] (mandatory): The endpoint of the provider's search interface + * **need_auth** [bool]: if authentication is needed for the search request; default: False + * **ssl_verify** [bool]: if the ssl certificates should be verified in requests; default: True + * **dont_quote** [List[str]]: characters that should not be quoted in the url params + * **literal_search_params** [Dict[str, str]]: A mapping of (search_param => search_value) pairs + giving search parameters to be passed as is in the search url query string. This is useful + for example in situations where the user wants to add a fixed search query parameter exactly + as it is done on the provider interface. + * **pagination** [:class:`~eodag.config.PluginConfig.Pagination`] (mandatory): The configuration of + how the pagination is done on the provider. It is a tree with the following nodes: + + * **next_page_url_tpl** [str] (mandatory): The template for pagination requests. This is a simple + Python format string which will be resolved using the following keywords: + ``url`` (the base url of the search endpoint), ``search`` (the query string corresponding + to the search request), ``items_per_page`` (the number of items to return per page), + ``skip`` (the number of items to skip) or ``skip_base_1`` (the number of items to skip, + starting from 1) and ``page`` (which page to return). + * **total_items_nb_key_path** [str]: An XPath or JsonPath leading to the total number of + results satisfying a request. This is used for providers which provides the total results + metadata along with the result of the query and don't have an endpoint for querying + the number of items satisfying a request, or for providers for which the count endpoint + returns a json or xml document + * **count_endpoint** [str]: The endpoint for counting the number of items satisfying a request + * **count_tpl** [str]: template for the count parameter that should be added to the search request + * **next_page_url_key_path** [str]: A JSONPATH expression used to retrieve the URL of the next page in the response of the current page. + * **max_items_per_page** [int]: The maximum number of items per page that the provider can + handle; default: 50 + * **discover_product_types** [Dict[str, Any]]: configuration for product type discovery + based on information from the provider; It contains the keys: + + * **fetch_url** [str] (mandatory): url from which the product types can be fetched + * **result_type** [str]: type of the provider result; currently only "json" supported + * **results_entry** [str] (mandatory): json path to the list of product types + * **generic_product_type_id** [str]: mapping for the product type id + * **generic_product_type_parsable_metadata** [Dict[str, str]]: mapping for product type + metadata (e.g. abstract, licence) which can be parsed from the provider result + * **generic_product_type_parsable_properties** [Dict[str, str]]: mapping for product type + properties which can be parsed from the result that are not product type metadata + * **single_collection_fetch_url** [str]: url to fetch data for a single collection; + used if product type metadata is not available from the endpoint given in `fetch_url` + * **single_collection_fetch_qs** [str]: query string to be added to the `fetch_url` + to filter for a collection + * **single_product_type_parsable_metadata** [Dict[str, str]]: mapping for product type + metadata returned by the endpoint given in `single_collection_fetch_url`. + + * **sort** [:class:`~eodag.config.PluginConfig.Sort`]: configuration for sorting the + results. It contains the keys: + + * **sort_by_default** [List[Tuple(str, Literal["ASC", "DESC"])]]: parameter and sort order by + which the result will be sorted by default (if the user does not enter a sort_by parameter); + if not given the result will use the default sorting of the provider; Attention: for some + providers sorting might cause a timeout if no filters are used. In that case no default + sort parameters should be given. The format is:: + + sort_by_default: + - !!python/tuple [, (ASC or DESC)] + + * **sort_by_tpl** [str]: template for the sort parameter that is added to the request; + It contains the parameters `sort_param` and `sort_order` which will be replaced by user input + or default value. If the parameters are added as query params to a GET request, the string + should start with '&', otherwise it should be a valid json string surrounded by '{{ }}'. + * **sort_param_mapping** [Dict [str, str]]: mapping for the parameters available for sorting + * **sort_order_mapping** [Dict[Literal["ascending", "descending"], str]]: mapping for + the sort order + * **max_sort_params** [int]: maximum number of sort parameters; used to validate the user + input; The tuples given by the user will be concatenated so, if no max is given and + the user input has more parameters than allowed by the provider, an invalid request is created. + + * **metadata_mapping** [Dict[str, Any]]: The search plugins of this kind can detect when a + metadata mapping is "query-able", and get the semantics of how to format the query string + parameter that enables to make a query on the corresponding metadata. To make a metadata query-able, + just configure it in the metadata mapping to be a list of 2 items, the first one being the + specification of the query string search formatting. The later is a string following the + specification of Python string formatting, with a special behaviour added to it. For example, + an entry in the metadata mapping of this kind:: - - **free_text_search_operations**: (optional) A tree structure of the form:: - - # noqa: E800 - : # e.g: $search - union: # how to join the operations below (e.g: ' AND ' --> - # '(op1 AND op2) AND (op3 OR op4)') - wrapper: # a pattern for how each operation will be wrapped - # (e.g: '({})' --> '(op1 AND op2)') - operations: # The operations to build - : # e.g: AND - - # e.g: - # 'sensingStartDate:[{startTimeFromAscendingNode}Z TO *]' - - # e.g: - # 'sensingStopDate:[* TO {completionTimeFromAscendingNode}Z]' - ... - ... - ... + completionTimeFromAscendingNode: + - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}' + - '$.properties.acquisition.endViewingDate' + + means that the search url will have a query string parameter named *"f"* with a value of + *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done with the value + of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that + ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value + of ``completionTimeFromAscendingNode``. This example shows all there is to know about the + semantics of the query string formatting introduced by this plugin: any eodag search parameter + can be referenced in the query string with an additional optional conversion function that + is separated from it by a ``#`` (see :func:`~eodag.utils.format_metadata` for further details + on the available converters). Note that for the values in the ``free_text_search_operations`` + configuration parameter follow the same rule. If the metadata_mapping is not a list but + only a string, this means that the parameters is not queryable but it is included in the + result obtained from the provider. The string indicates how the provider result should be + mapped to the eodag parameter. + * **discover_metadata** [:class:`~eodag.config.PluginConfig.DiscoverMetadata`]: configuration + for the auto-discovery of queryable parameters as well as parameters returned by the provider + which are not in the metadata mapping. It has the attributes: + + * **auto_discovery** [bool]: if the automatic discovery of metadata is activated; default: + False; if false, the other parameters are not used; + * **metadata_pattern** [str]: regex string a parameter in the result should match so + that is used + * **search_param** [Union [str, Dict[str, Any]]]: format to add a query param given by + the user and not in the metadata mapping to the requets, 'metadata' will be replaced by + the search param; can be a string or a dict containing `free_text_search_operations` (see below) + * **metadata_path** [str]: path where the queryable properties can be found in the provider result + + * **free_text_search_operations**: (optional) A tree structure of the form:: + + # noqa: E800 + : # e.g: $search + union: # how to join the operations below (e.g: ' AND ' --> + # '(op1 AND op2) AND (op3 OR op4)') + wrapper: # a pattern for how each operation will be wrapped + # (e.g: '({})' --> '(op1 AND op2)') + operations: # The operations to build + : # e.g: AND + - # e.g: + # 'sensingStartDate:[{startTimeFromAscendingNode}Z TO *]' + - # e.g: + # 'sensingStopDate:[* TO {completionTimeFromAscendingNode}Z]' + ... + ... + ... With the structure above, each operation will become a string of the form: '( )', then the operations will be joined together using the union string and finally if the number of operations is greater than 1, they will be wrapped as specified by the wrapper config key. - The search plugins of this kind can detect when a metadata mapping is "query-able", - and get the semantics of how to format the query string parameter that enables to - make a query on the corresponding metadata. To make a metadata query-able, just - configure it in the metadata mapping to be a list of 2 items, the first one being - the specification of the query string search formatting. The later is a string - following the specification of Python string formatting, with a special behaviour - added to it. For example, an entry in the metadata mapping of this kind:: - - completionTimeFromAscendingNode: - - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}' - - '$.properties.acquisition.endViewingDate' - - means that the search url will have a query string parameter named *"f"* with a - value of *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done - with the value of ``completionTimeFromAscendingNode`` being - ``2018-12-04T12:18:00``. What happened is that - ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp - of the value of ``completionTimeFromAscendingNode``. This example shows all there - is to know about the semantics of the query string formatting introduced by this - plugin: any eodag search parameter can be referenced in the query string - with an additional optional conversion function that is separated from it by a - ``#`` (see :func:`~eodag.utils.format_metadata` for further details on the - available converters). Note that for the values in the - ``free_text_search_operations`` configuration parameter follow the same rule. - - :param provider: An eodag providers configuration dictionary - :param config: Path to the user configuration file """ extract_properties: Dict[str, Callable[..., Dict[str, Any]]] = { diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml index 849094cc0..19ca0c3e1 100644 --- a/eodag/resources/providers.yml +++ b/eodag/resources/providers.yml @@ -1599,7 +1599,7 @@ startTimeFromAscendingNode: properties.datetime creationDate: properties.created metadata_mapping: - # redefine the following mapppings as the provider does not support advanced queries/filtering, + # redefine the following mappings as the provider does not support advanced queries/filtering, # these parameters will not be queryable doi: '$.properties."sci:doi"' processingLevel: '$.properties."processing:level"' From 6db1516a9e21f0ed03613ea6e5a52d229882e5d3 Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Tue, 13 Aug 2024 17:13:32 +0200 Subject: [PATCH 05/25] docs: configuration parameters for search plugins --- eodag/plugins/download/http.py | 4 +- eodag/plugins/search/build_search_result.py | 35 +++------ eodag/plugins/search/cop_marine.py | 11 ++- eodag/plugins/search/creodias_s3.py | 6 +- eodag/plugins/search/qssearch.py | 79 ++++++++++++++++++--- eodag/plugins/search/static_stac_search.py | 22 +++--- 6 files changed, 107 insertions(+), 50 deletions(-) diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py index 593d82846..0ddc370c1 100644 --- a/eodag/plugins/download/http.py +++ b/eodag/plugins/download/http.py @@ -113,7 +113,9 @@ class HTTPDownload(Download): * **ssl_verify** [bool]: if the ssl certificates should be verified in requests; default: True * **output_extension** [str]: which extension should be used for the downloaded file * **no_auth_download** [bool]: if the download should be done without authentication; default: True - * **order_enabled** [bool]: if the product has to be ordered to download it; default: False + * **order_enabled** [bool]: if the product has to be ordered to download it; if this paramter + is set to true, a mapping for the orderLink has to be added to the metadata mapping of + the search plugin used for the provider; default: False * **order_method** [str]: HTTP request method for the order request (GET or POST); default: GET * **order_headers** [Dict[str, str]]: headers to be added to the order request * **order_on_response** [:class:`~eodag.config.PluginConfig.OrderOnResponse`]: a typed dictionary diff --git a/eodag/plugins/search/build_search_result.py b/eodag/plugins/search/build_search_result.py index 607f5a4e1..b44c6f449 100644 --- a/eodag/plugins/search/build_search_result.py +++ b/eodag/plugins/search/build_search_result.py @@ -81,21 +81,12 @@ class BuildPostSearchResult(PostJsonSearch): performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult` object. - The available configuration parameters inherits from parent classes, with particularly - for this plugin: + The available configuration parameters inherits from parent classes (PostJsonSearch and + QueryStringSearch), with particularly for this plugin: - - **api_endpoint**: (mandatory) The endpoint of the provider's search interface + * **remove_from_query** [List[str]]: List of parameters used to parse metadata but that must + not be included to the query - - **pagination**: The configuration of how the pagination is done - on the provider. It is a tree with the following nodes: - - - *next_page_query_obj*: (optional) The additional parameters needed to perform - search. These paramaters won't be included in result. This must be a json dict - formatted like `{{"foo":"bar"}}` because it will be passed to a `.format()` - method before being loaded as json. - - :param provider: An eodag providers configuration dictionary - :param config: Path to the user configuration file """ def count_hits( @@ -188,7 +179,7 @@ def normalize_results( result.update(results.product_type_def_params) result = dict(result, **{k: v for k, v in kwargs.items() if v is not None}) - # parse porperties + # parse properties parsed_properties = properties_from_json( result, self.config.metadata_mapping, @@ -249,19 +240,13 @@ class BuildSearchResult(BuildPostSearchResult): This plugin builds a single :class:`~eodag.api.search_result.SearchResult` object using given query parameters as product properties. - The available configuration parameters inherits from parent classes, with particularly - for this plugin: - - - **end_date_excluded**: Set to `False` if provider does not include end date to - search - - - **remove_from_query**: List of parameters used to parse metadata but that must - not be included to the query + The available configuration parameters inherits from parent classes (BuildPostSearchResult, + PostJsonSearch and QueryStringSearch), with particularly for this plugin: - - **constraints_file_url**: url of the constraint file used to build queryables + * **end_date_excluded** [bool]: Set to `False` if provider does not include end date in + the search request; In this case, if the end date is at midnight, the previous day will be + used. default: true - :param provider: An eodag providers configuration dictionary - :param config: Path to the user configuration file """ def __init__(self, provider: str, config: PluginConfig) -> None: diff --git a/eodag/plugins/search/cop_marine.py b/eodag/plugins/search/cop_marine.py index 99afaab09..cec633726 100644 --- a/eodag/plugins/search/cop_marine.py +++ b/eodag/plugins/search/cop_marine.py @@ -109,7 +109,16 @@ def _check_int_values_properties(properties: Dict[str, Any]): class CopMarineSearch(StaticStacSearch): - """class that implements search for the Copernicus Marine provider""" + """class that implements search for the Copernicus Marine provider + It calls discover_product_types inherited from StaticStacSearch but for the actual search + a special method which fetches the urls of the available products from an S3 storage and + filters them has been written. + The configuration parameters are inherited from the parent and grand-parent classes. The + `auto_discovery` parameter in the `discover_metadata` section has to be set to `false` and the + `fetch_url` in the `discover_queryables` queryables section has to be set to `null` to + overwrite the default config from the stac provider configuration because those functionalities + are not available. + """ def __init__(self, provider: str, config: PluginConfig): original_metadata_mapping = copy.deepcopy(config.metadata_mapping) diff --git a/eodag/plugins/search/creodias_s3.py b/eodag/plugins/search/creodias_s3.py index 0f53694a5..d47fd6a42 100644 --- a/eodag/plugins/search/creodias_s3.py +++ b/eodag/plugins/search/creodias_s3.py @@ -117,7 +117,11 @@ def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth): class CreodiasS3Search(ODataV4Search): """ - Search on creodias and adapt results to s3 + CreodiasS3Search is an extension of ODataV4Search, it executes a Search on creodias and + adapts results so that the assets contain links to s3. It has the same configuration + parameters as ODataV4Search and one additional parameter: + + * **s3_endpoint** [str] (mandatory): base url of the s3 """ def __init__(self, provider, config): diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index 66e9317aa..b7ffd9a95 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -120,8 +120,11 @@ class QueryStringSearch(Search): result that gives access to the result entries * **api_endpoint** [str] (mandatory): The endpoint of the provider's search interface * **need_auth** [bool]: if authentication is needed for the search request; default: False + * **auth_error_code** [int]: which error code is returned in case of an authentication + error; only used if `need_auth=true` * **ssl_verify** [bool]: if the ssl certificates should be verified in requests; default: True * **dont_quote** [List[str]]: characters that should not be quoted in the url params + * **timeout** [int]: time to wait until request timeout in seconds; default: 5 * **literal_search_params** [Dict[str, str]]: A mapping of (search_param => search_value) pairs giving search parameters to be passed as is in the search url query string. This is useful for example in situations where the user wants to add a fixed search query parameter exactly @@ -146,11 +149,13 @@ class QueryStringSearch(Search): the URL of the next page in the response of the current page. * **max_items_per_page** [int]: The maximum number of items per page that the provider can handle; default: 50 + * **discover_product_types** [Dict[str, Any]]: configuration for product type discovery based on information from the provider; It contains the keys: * **fetch_url** [str] (mandatory): url from which the product types can be fetched - * **result_type** [str]: type of the provider result; currently only "json" supported + * **result_type** [str]: type of the provider result; currently only "json" is supported, + (other types could be used in an extension of this plugin) * **results_entry** [str] (mandatory): json path to the list of product types * **generic_product_type_id** [str]: mapping for the product type id * **generic_product_type_parsable_metadata** [Dict[str, str]]: mapping for product type @@ -183,9 +188,9 @@ class QueryStringSearch(Search): * **sort_param_mapping** [Dict [str, str]]: mapping for the parameters available for sorting * **sort_order_mapping** [Dict[Literal["ascending", "descending"], str]]: mapping for the sort order - * **max_sort_params** [int]: maximum number of sort parameters; used to validate the user - input; The tuples given by the user will be concatenated so, if no max is given and - the user input has more parameters than allowed by the provider, an invalid request is created. + * **max_sort_params** [int]: maximum number of sort parameters supported by the provider; + used to validate the user input to avoid failed requests or unexpected behaviour + (not all parameters are used in the request) * **metadata_mapping** [Dict[str, Any]]: The search plugins of this kind can detect when a metadata mapping is "query-able", and get the semantics of how to format the query string @@ -248,6 +253,24 @@ class QueryStringSearch(Search): the union string and finally if the number of operations is greater than 1, they will be wrapped as specified by the wrapper config key. + * **discover_queryables** [Dict[str, Any]]: configuration to fetch the queryables from a + provider queryables endpoint; It has the following keys: + + * **fetch_url** [str]: url to fetch the queryables valid for all product types + * **product_type_fetch_url** [str]: url to fetch the queryables for a specific product type + * **result_type** [str]: type of the result (currently only json is used) + * **results_entry** [str]: json path to retrieve the queryables from the provider result + + * **constraints_file_url** [str]: url to fetch the constraints for a specific product type, + can be an http url or a path to a file; the constraints are used to build queryables + * **constraints_file_dataset_key** [str]: key which is used in the eodag configuration to + map the eodag product type to the provider product type; default: dataset + * **constraints_entry** [str]: key in the json result where the constraints can be found; + if not given, it is assumed that the constraints are on tope level of the result, i.e. + the result is an array of constraints + * **stop_without_constraints_entry_key** [bool]: if true only a provider result containing + `constraints_entry` is accepted as valid and used to create constraints; default: false + """ extract_properties: Dict[str, Callable[..., Dict[str, Any]]] = { @@ -1229,7 +1252,20 @@ def _request( class ODataV4Search(QueryStringSearch): """A specialisation of a QueryStringSearch that does a two step search to retrieve - all products metadata""" + all products metadata. All configuration parameters of QueryStringSearch are also available + for this plugin. In addition, the following parameters can be configured: + + * **per_product_metadata_query** [bool]: should be set to true if the metadata is not given + in the search result and a two step search has to be performed; default: false + * **metadata_pre_mapping** [Dict[str, str]]: a dictionary which can be used to simplify + further metadata extraction. For example, going from '$.Metadata[?(@.id="foo")].value' + to '$.Metadata.foo.value'. It has the keys: + + * **metadata_path** [str]: json path of the metadata entry + * **metadata_path_id** [str]: key to get the metadata id + * **metadata_path_value** [str]: key to get the metadata value + + """ def __init__(self, provider: str, config: PluginConfig) -> None: super(ODataV4Search, self).__init__(provider, config) @@ -1320,7 +1356,26 @@ def normalize_results( class PostJsonSearch(QueryStringSearch): - """A specialisation of a QueryStringSearch that uses POST method""" + """A specialisation of a QueryStringSearch that uses POST method + All configuration parameters available for QueryStringSearch are also available for PostJsonSearch. + The mappings given in metadata_mapping are used to construct a (json) body for the + POST request that is sent to the provider. Due to the fact that we sent a POST request and + not a get request, the pagination configuration will look slightly different. It has the + following parameters: + + * **next_page_query_obj**: The additional parameters needed to add pagination information to + the search request. These parameters won't be included in result. This must be a json dict + formatted like `{{"foo":"bar"}}` because it will be passed to a `.format()` method + before being loaded as json. + * **total_items_nb_key_path** [str]: An XPath or JsonPath leading to the total number of + results satisfying a request. This is used for providers which provides the total results + metadata along with the result of the query and don't have an endpoint for querying + the number of items satisfying a request, or for providers for which the count endpoint + returns a json or xml document + * **max_items_per_page** [int]: The maximum number of items per page that the provider can + handle; default: 50 + + """ def _get_default_end_date_from_start_date( self, start_datetime: str, product_type: str @@ -1704,7 +1759,15 @@ def _request( class StacSearch(PostJsonSearch): - """A specialisation of a QueryStringSearch that uses generic STAC configuration""" + """A specialisation of PostJsonSearch that uses generic STAC configuration, it therefore + has the same configuration parameters (those inherited from QuerySTringSearch) + For providers using StacSearch default values are defined for most of the parameters + (see :doc:`eodag/resources/stac_provider.yml`). If some parameters are different for a + specific provider, they have to be overwritten. If certain functionalities are not available, + their configuration parameters have to be overwritten with `null`. E.g. if there is no queryables + endpoint, the `fetch_url` and `product_type_fetch_url` in the `discover_queryables` config have + to be set to `null`. + """ def __init__(self, provider: str, config: PluginConfig) -> None: # backup results_entry overwritten by init @@ -1855,7 +1918,7 @@ def discover_queryables( class PostJsonSearchWithStacQueryables(StacSearch, PostJsonSearch): """A specialisation of a :class:`~eodag.plugins.search.qssearch.PostJsonSearch` that - uses generic STAC configuration for queryables. + uses generic STAC configuration for queryables (inherited from StacSearch). """ def __init__(self, provider: str, config: PluginConfig) -> None: diff --git a/eodag/plugins/search/static_stac_search.py b/eodag/plugins/search/static_stac_search.py index 277e93e5c..76f1c721b 100644 --- a/eodag/plugins/search/static_stac_search.py +++ b/eodag/plugins/search/static_stac_search.py @@ -43,24 +43,18 @@ class StaticStacSearch(StacSearch): """Static STAC Catalog search plugin - - The available configuration parameters for this plugin are - (to be set in provider configuration): - - - **api_endpoint**: (mandatory) path to the catalog (url or local system path) - - - **max_connections**: (optional) Maximum number of connections for HTTP requests, - defaut is 100. - - - **timeout**: (mandatory) Timeout in seconds for each internal HTTP request, - default is 5. - This plugin first loads all STAC items found in the catalog, and converts them to EOProducts using StacSearch. Then it uses crunchers to only keep products matching query parameters. - :param provider: An eodag providers configuration dictionary - :param config: Path to the user configuration file + The plugin inherits the configuration parameters from PostJsonSearch (via the StacSearch + inheritance) with the following particularities: + + * **api_endpoint** [str] (mandatory): path to the catalog; in contrast to the api_endpoint for + other plugin types this can be a url or local system path. + * **max_connections** [int]: Maximum number of connections for HTTP requests; default: 100. + * **timeout** [int]: Timeout in seconds for each internal HTTP request; default: 5 + """ def __init__(self, provider: str, config: PluginConfig) -> None: From 6c65dc2f54c2be989298f0b1ffcc98212391a4c2 Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Wed, 14 Aug 2024 11:38:23 +0200 Subject: [PATCH 06/25] docs: configuration params for various plugins --- .../plugins/authentication/openid_connect.py | 9 ++- eodag/plugins/crunch/base.py | 5 +- eodag/plugins/crunch/filter_date.py | 6 +- .../plugins/crunch/filter_latest_intersect.py | 3 +- .../plugins/crunch/filter_latest_tpl_name.py | 4 +- eodag/plugins/crunch/filter_overlap.py | 12 +-- eodag/plugins/crunch/filter_property.py | 6 +- eodag/plugins/download/s3rest.py | 25 +++--- eodag/plugins/search/base.py | 2 + eodag/plugins/search/csw.py | 38 ++++++++- eodag/plugins/search/data_request_search.py | 81 ++++++++++++++++++- eodag/plugins/search/qssearch.py | 1 + 12 files changed, 157 insertions(+), 35 deletions(-) diff --git a/eodag/plugins/authentication/openid_connect.py b/eodag/plugins/authentication/openid_connect.py index 5bb782bfc..4ffce2156 100644 --- a/eodag/plugins/authentication/openid_connect.py +++ b/eodag/plugins/authentication/openid_connect.py @@ -42,9 +42,12 @@ class OIDCRefreshTokenBase(Authentication): - """OIDC refresh token base class, to be used through specific OIDC flows plugins. + """OIDC refresh token base class, to be used through specific OIDC flows plugins; + Common mechanism to handle refresh token from all OIDC auth plugins; + Plugins inheriting from this base class must implement the methods _request_new_token and + _get_token_with_refresh_token. Depending oh the implementation of these methods they can have + different configuration parameters. - Common mechanism to handle refresh token from all OIDC auth plugins. """ class TokenInfo(TypedDict, total=False): @@ -186,7 +189,7 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase): * **authorization_uri** [str] (mandatory): The authorization url of the server (where to query for grants) * **redirect_uri** [str] (mandatory): The callback url that will handle the code given by the OIDC provider * **token_uri** [str] (mandatory): The url to query to exchange the authorization - code obtained from the OIDC providerfor an authorized token + code obtained from the OIDC provider for an authorized token * **client_id** [str] (mandatory): The OIDC provider's client ID of the eodag provider * **user_consent_needed** [bool] (mandatory): Whether a user consent is needed during the authentication diff --git a/eodag/plugins/crunch/base.py b/eodag/plugins/crunch/base.py index db2f16265..5bb367c32 100644 --- a/eodag/plugins/crunch/base.py +++ b/eodag/plugins/crunch/base.py @@ -27,7 +27,10 @@ class Crunch(PluginTopic): - """Base cruncher""" + """Base cruncher + :param config: Crunch configuration + :type config: Dict[str, Any] + """ def __init__(self, config: Optional[Dict[str, Any]]) -> None: self.config = PluginConfig() diff --git a/eodag/plugins/crunch/filter_date.py b/eodag/plugins/crunch/filter_date.py index 6901fd855..e52535693 100644 --- a/eodag/plugins/crunch/filter_date.py +++ b/eodag/plugins/crunch/filter_date.py @@ -37,10 +37,10 @@ class FilterDate(Crunch): """FilterDate cruncher: filter products by date - :param config: Crunch configuration, may contain : + The Crunch configuration, may contain : - - `start`: (optional) start sensing time in iso format - - `end`: (optional) end sensing time in iso format + * **start** [str]: start sensing time in iso format + * **end** [str]: end sensing time in iso format """ @staticmethod diff --git a/eodag/plugins/crunch/filter_latest_intersect.py b/eodag/plugins/crunch/filter_latest_intersect.py index 3b91989e9..aaa404f77 100644 --- a/eodag/plugins/crunch/filter_latest_intersect.py +++ b/eodag/plugins/crunch/filter_latest_intersect.py @@ -39,7 +39,8 @@ class FilterLatestIntersect(Crunch): """FilterLatestIntersect cruncher - Filter latest products (the ones with a the highest start date) that intersect search extent + Filter latest products (the ones with a the highest start date) that intersect search extent; + The configuration for this plugin is an empty dict """ @staticmethod diff --git a/eodag/plugins/crunch/filter_latest_tpl_name.py b/eodag/plugins/crunch/filter_latest_tpl_name.py index 669943bf6..53dd9c1d2 100644 --- a/eodag/plugins/crunch/filter_latest_tpl_name.py +++ b/eodag/plugins/crunch/filter_latest_tpl_name.py @@ -35,9 +35,9 @@ class FilterLatestByName(Crunch): Filter Search results to get only the latest product, based on the name of the product - :param config: Crunch configuration, must contain : + The Crunch configuration must contain : - - `name_pattern` : product name pattern + * **name_pattern** [str] (mandatory) : product name pattern """ NAME_PATTERN_CONSTRAINT = re.compile(r"\(\?P\\d\{6\}\)") diff --git a/eodag/plugins/crunch/filter_overlap.py b/eodag/plugins/crunch/filter_overlap.py index 43b7fbc0d..b5a7d25f6 100644 --- a/eodag/plugins/crunch/filter_overlap.py +++ b/eodag/plugins/crunch/filter_overlap.py @@ -40,14 +40,14 @@ class FilterOverlap(Crunch): Filter products, retaining only those that are overlapping with the search_extent - :param config: Crunch configuration, may contain : + The Crunch configuration may contain : - - `minimum_overlap` : minimal overlap percentage - - `contains` : True if product geometry contains the search area - - `intersects` : True if product geometry intersects the search area - - `within` : True if product geometry is within the search area + * **minimum_overlap** [Union[float, str]]: minimal overlap percentage; default: "0" + * **contains** [bool]: True if product geometry contains the search area; default: False + * **intersects** [bool]: True if product geometry intersects the search area; default: False + * **within** [bool]: True if product geometry is within the search area; default: False - These configuration parameters are mutually exclusive. + These configuration parameters are mutually exclusive. """ def proceed( diff --git a/eodag/plugins/crunch/filter_property.py b/eodag/plugins/crunch/filter_property.py index 2bb064e79..4e70ffd93 100644 --- a/eodag/plugins/crunch/filter_property.py +++ b/eodag/plugins/crunch/filter_property.py @@ -34,10 +34,10 @@ class FilterProperty(Crunch): Filter products, retaining only those whose property match criteria - :param config: Crunch configuration, should contain : + The Crunch configuration should contain : - - `property=value` : property key from product.properties, associated to its filter value - - `operator` : (optional) Operator used for filtering (one of `lt,le,eq,ne,ge,gt`). Default is `eq` + * **** [Any] (mandatory): property key from product.properties, associated to its filter value + * **operator** [str]: Operator used for filtering (one of `lt,le,eq,ne,ge,gt`). Default is `eq` """ def proceed( diff --git a/eodag/plugins/download/s3rest.py b/eodag/plugins/download/s3rest.py index 79dc5517b..43de0e4e0 100644 --- a/eodag/plugins/download/s3rest.py +++ b/eodag/plugins/download/s3rest.py @@ -67,18 +67,19 @@ class S3RestDownload(Download): Re-use AwsDownload bucket some handling methods - :param provider: provider name - :param config: Download plugin configuration: - - * ``config.base_uri`` (str) - default endpoint url - * ``config.extract`` (bool) - (optional) extract downloaded archive or not - * ``config.auth_error_code`` (int) - (optional) authentication error code - * ``config.bucket_path_level`` (int) - (optional) bucket location index in path.split('/') - * ``config.order_enabled`` (bool) - (optional) wether order is enabled or not if product is `OFFLINE` - * ``config.order_method`` (str) - (optional) HTTP request method, GET (default) or POST - * ``config.order_headers`` (dict) - (optional) order request headers - * ``config.order_on_response`` (dict) - (optional) edit or add new product properties - * ``config.order_status`` (:class:`~eodag.config.PluginConfig.OrderStatus`) - Order status handling + Download plugin configuration: + + * **config.base_uri** [str] (mandatory): default endpoint url + * **config.extract** [bool]: extract downloaded archive or not + * **config.auth_error_code** [int]: authentication error code + * **config.bucket_path_level** [int]: bucket location index in path.split('/') + * **config.order_enabled** [bool]: whether order is enabled or not if product is `OFFLINE` + * **config.order_method** [str]: HTTP request method, GET (default) or POST + * **config.order_headers** [dict]: order request headers + * **order_on_response** [:class:`~eodag.config.PluginConfig.OrderOnResponse`]: a typed dictionary + containing the key 'metadata_mapping' which can be used to add new product properties + based on the data in response to the order request + * **config.order_status** [:class:`~eodag.config.PluginConfig.OrderStatus`]: Order status handling """ def __init__(self, provider: str, config: PluginConfig) -> None: diff --git a/eodag/plugins/search/base.py b/eodag/plugins/search/base.py index 035005221..29d489c71 100644 --- a/eodag/plugins/search/base.py +++ b/eodag/plugins/search/base.py @@ -57,7 +57,9 @@ class Search(PluginTopic): """Base Search Plugin. :param provider: An EODAG provider name + :type provider: str :param config: An EODAG plugin configuration + :type config: Dict[str, Any] """ auth: Union[AuthBase, Dict[str, str]] diff --git a/eodag/plugins/search/csw.py b/eodag/plugins/search/csw.py index e12fee62f..214da4f72 100644 --- a/eodag/plugins/search/csw.py +++ b/eodag/plugins/search/csw.py @@ -52,7 +52,43 @@ class CSWSearch(Search): - """A plugin for implementing search based on OGC CSW""" + """A plugin for implementing search based on OGC CSW + It has the following configuration parameters: + + * **api_endpoint** [str] (mandatory): The endpoint of the provider's search interface + * **version** [str]: OGC Catalogue Service version; default: 2.0.2 + * **search_definition** [Dict[str, Any]] (mandatory): + + * **product_type_tags** [List[Dict[str, Any]]: dict of product type tags + * **resource_location_filter** [str]: regex string + * **date_tags** [Dict[str, Any]]: tags for start and end + + * **metadata_mapping** [Dict[str, Any]]: The search plugins of this kind can detect when a + metadata mapping is "query-able", and get the semantics of how to format the query string + parameter that enables to make a query on the corresponding metadata. To make a metadata query-able, + just configure it in the metadata mapping to be a list of 2 items, the first one being the + specification of the query string search formatting. The later is a string following the + specification of Python string formatting, with a special behaviour added to it. For example, + an entry in the metadata mapping of this kind:: + + completionTimeFromAscendingNode: + - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}' + - '$.properties.acquisition.endViewingDate' + + means that the search url will have a query string parameter named *"f"* with a value of + *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done with the value + of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that + ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value + of ``completionTimeFromAscendingNode``. This example shows all there is to know about the + semantics of the query string formatting introduced by this plugin: any eodag search parameter + can be referenced in the query string with an additional optional conversion function that + is separated from it by a ``#`` (see :func:`~eodag.utils.format_metadata` for further details + on the available converters). Note that for the values in the ``free_text_search_operations`` + configuration parameter follow the same rule. If the metadata_mapping is not a list but + only a string, this means that the parameters is not queryable but it is included in the + result obtained from the provider. The string indicates how the provider result should be + mapped to the eodag parameter. + """ def __init__(self, provider: str, config: PluginConfig) -> None: super(CSWSearch, self).__init__(provider, config) diff --git a/eodag/plugins/search/data_request_search.py b/eodag/plugins/search/data_request_search.py index 862abd4ec..89039f265 100644 --- a/eodag/plugins/search/data_request_search.py +++ b/eodag/plugins/search/data_request_search.py @@ -58,9 +58,84 @@ class DataRequestSearch(Search): """ Plugin to execute search requests composed of several steps: - - do a data request which defines which data shall be searched - - check the status of the request job - - if finished - fetch the result of the job + + #. do a data request which defines which data shall be searched + #. check the status of the request job + #. if finished - fetch the result of the job + + It has the following configuration parameters: + + * **api_endpoint** [str] (mandatory): The endpoint of the provider's search interface + * **results_entry** [str] (mandatory): The name of the key in the provider search + result that gives access to the result entries + * **data_request_url** [str] (mandatory): url to which the data request shall be sent + * **status_url** [str] (mandatory): url to fetch the status of the data request + * **result_url** [str] (mandatory): url to fetch the search result when the data request is done + * **need_auth** [bool]: if authentication is needed for the search request; default: False + * **auth_error_code** [int]: which error code is returned in case of an authentication + error; only used if `need_auth=true` + * **ssl_verify** [bool]: if the ssl certificates should be verified in requests; default: True + * **timeout** [int]: time to wait until request timeout in seconds; default: 5 + * **dates_required** [bool]: if date parameters are mandatory in the request; default: True + * **pagination** [:class:`~eodag.config.PluginConfig.Pagination`] (mandatory): The configuration of + how the pagination is done on the provider. It is a tree with the following nodes: + + * **total_items_nb_key_path** [str]: An XPath or JsonPath leading to the total number of + results satisfying a request. This is used for providers which provides the total results + metadata along with the result of the query and don't have an endpoint for querying + the number of items satisfying a request, or for providers for which the count endpoint + returns a json or xml document + * **max_items_per_page** [int]: The maximum number of items per page that the provider can + handle; default: 50 + * **start_page** [int]: number of the first page; default: 1 + * **discover_product_types** [Dict[str, Any]]: configuration for product type discovery + based on information from the provider; It contains the keys: + + * **fetch_url** [str] (mandatory): url from which the product types can be fetched + * **result_type** [str]: type of the provider result; currently only "json" is supported, + (other types could be used in an extension of this plugin) + * **results_entry** [str] (mandatory): json path to the list of product types + * **generic_product_type_id** [str]: mapping for the product type id + * **generic_product_type_parsable_metadata** [Dict[str, str]]: mapping for product type + metadata (e.g. abstract, licence) which can be parsed from the provider result + * **generic_product_type_parsable_properties** [Dict[str, str]]: mapping for product type + properties which can be parsed from the result that are not product type metadata + * **single_collection_fetch_url** [str]: url to fetch data for a single collection; + used if product type metadata is not available from the endpoint given in `fetch_url` + * **single_collection_fetch_qs** [str]: query string to be added to the `fetch_url` + to filter for a collection + * **single_product_type_parsable_metadata** [Dict[str, str]]: mapping for product type + metadata returned by the endpoint given in `single_collection_fetch_url`. + * **constraints_file_url** [str]: url to fetch the constraints for a specific product type, + can be an http url or a path to a file; the constraints are used to build queryables + * **constraints_entry** [str]: key in the json result where the constraints can be found; + if not given, it is assumed that the constraints are on tope level of the result, i.e. + the result is an array of constraints + * **metadata_mapping** [Dict[str, Any]]: The search plugins of this kind can detect when a + metadata mapping is "query-able", and get the semantics of how to format the query string + parameter that enables to make a query on the corresponding metadata. To make a metadata query-able, + just configure it in the metadata mapping to be a list of 2 items, the first one being the + specification of the query string search formatting. The later is a string following the + specification of Python string formatting, with a special behaviour added to it. For example, + an entry in the metadata mapping of this kind:: + + completionTimeFromAscendingNode: + - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}' + - '$.properties.acquisition.endViewingDate' + + means that the search url will have a query string parameter named *"f"* with a value of + *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done with the value + of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that + ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value + of ``completionTimeFromAscendingNode``. This example shows all there is to know about the + semantics of the query string formatting introduced by this plugin: any eodag search parameter + can be referenced in the query string with an additional optional conversion function that + is separated from it by a ``#`` (see :func:`~eodag.utils.format_metadata` for further details + on the available converters). Note that for the values in the ``free_text_search_operations`` + configuration parameter follow the same rule. If the metadata_mapping is not a list but + only a string, this means that the parameters is not queryable but it is included in the + result obtained from the provider. The string indicates how the provider result should be + mapped to the eodag parameter. """ data_request_id: Optional[str] diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index b7ffd9a95..cb0e2208a 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -149,6 +149,7 @@ class QueryStringSearch(Search): the URL of the next page in the response of the current page. * **max_items_per_page** [int]: The maximum number of items per page that the provider can handle; default: 50 + * **start_page** [int]: number of the first page; default: 1 * **discover_product_types** [Dict[str, Any]]: configuration for product type discovery based on information from the provider; It contains the keys: From 90ff7cda169830b02d57c413c8ae0d5fc0035df3 Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Wed, 14 Aug 2024 14:43:50 +0200 Subject: [PATCH 07/25] docs: small fixes --- eodag/plugins/search/qssearch.py | 10 +++++----- eodag/plugins/search/static_stac_search.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index cb0e2208a..eed612749 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -1761,12 +1761,12 @@ def _request( class StacSearch(PostJsonSearch): """A specialisation of PostJsonSearch that uses generic STAC configuration, it therefore - has the same configuration parameters (those inherited from QuerySTringSearch) + has the same configuration parameters (those inherited from QueryStringSearch) For providers using StacSearch default values are defined for most of the parameters - (see :doc:`eodag/resources/stac_provider.yml`). If some parameters are different for a - specific provider, they have to be overwritten. If certain functionalities are not available, - their configuration parameters have to be overwritten with `null`. E.g. if there is no queryables - endpoint, the `fetch_url` and `product_type_fetch_url` in the `discover_queryables` config have + (see stac_provider.yml). If some parameters are different for a specific provider, they + have to be overwritten. If certain functionalities are not available, their configuration + parameters have to be overwritten with `null`. E.g. if there is no queryables endpoint, + the `fetch_url` and `product_type_fetch_url` in the `discover_queryables` config have to be set to `null`. """ diff --git a/eodag/plugins/search/static_stac_search.py b/eodag/plugins/search/static_stac_search.py index 76f1c721b..e24cc52c9 100644 --- a/eodag/plugins/search/static_stac_search.py +++ b/eodag/plugins/search/static_stac_search.py @@ -51,7 +51,7 @@ class StaticStacSearch(StacSearch): inheritance) with the following particularities: * **api_endpoint** [str] (mandatory): path to the catalog; in contrast to the api_endpoint for - other plugin types this can be a url or local system path. + other plugin types this can be a url or local system path. * **max_connections** [int]: Maximum number of connections for HTTP requests; default: 100. * **timeout** [int]: Timeout in seconds for each internal HTTP request; default: 5 From 6492947f26b81c6f999242608732d15e07861c68 Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Tue, 20 Aug 2024 17:39:49 +0200 Subject: [PATCH 08/25] docs: fix typos --- eodag/plugins/search/data_request_search.py | 2 +- eodag/plugins/search/qssearch.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eodag/plugins/search/data_request_search.py b/eodag/plugins/search/data_request_search.py index 89039f265..1ea9d9e4b 100644 --- a/eodag/plugins/search/data_request_search.py +++ b/eodag/plugins/search/data_request_search.py @@ -109,7 +109,7 @@ class DataRequestSearch(Search): * **constraints_file_url** [str]: url to fetch the constraints for a specific product type, can be an http url or a path to a file; the constraints are used to build queryables * **constraints_entry** [str]: key in the json result where the constraints can be found; - if not given, it is assumed that the constraints are on tope level of the result, i.e. + if not given, it is assumed that the constraints are on top level of the result, i.e. the result is an array of constraints * **metadata_mapping** [Dict[str, Any]]: The search plugins of this kind can detect when a metadata mapping is "query-able", and get the semantics of how to format the query string diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index eed612749..dc23ee627 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -267,7 +267,7 @@ class QueryStringSearch(Search): * **constraints_file_dataset_key** [str]: key which is used in the eodag configuration to map the eodag product type to the provider product type; default: dataset * **constraints_entry** [str]: key in the json result where the constraints can be found; - if not given, it is assumed that the constraints are on tope level of the result, i.e. + if not given, it is assumed that the constraints are on top level of the result, i.e. the result is an array of constraints * **stop_without_constraints_entry_key** [bool]: if true only a provider result containing `constraints_entry` is accepted as valid and used to create constraints; default: false From 77a52ba946b6d5239cfd7eb07f534d3e3f61c09f Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Fri, 27 Sep 2024 11:11:32 +0200 Subject: [PATCH 09/25] docs: httpdownload params styling --- docs/plugins.rst | 4 +- eodag/config.py | 14 ++++-- eodag/plugins/download/http.py | 82 ++++++++++++++++++++-------------- 3 files changed, 61 insertions(+), 39 deletions(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index 41c8a78fb..52abea5e2 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -261,5 +261,5 @@ Plugin configuration :members: :member-order: bysource :undoc-members: - :exclude-members: priority, products, product_type_config, yaml_loader, from_mapping, from_yaml, update, validate, - yaml_dumper, yaml_tag + :exclude-members: priority, product_type_config, yaml_loader, from_mapping, from_yaml, update, validate, yaml_dumper, + yaml_tag diff --git a/eodag/config.py b/eodag/config.py index bf9ae7433..da3a56bd6 100644 --- a/eodag/config.py +++ b/eodag/config.py @@ -340,12 +340,12 @@ class OrderStatus(TypedDict): s3_bucket: str #: :class:`~eodag.plugins.base.PluginTopic` Authentication error codes auth_error_code: Union[int, List[int]] + #: :class:`~eodag.plugins.base.PluginTopic` Time to wait until request timeout in seconds + timeout: float # search & api ----------------------------------------------------------------------------------------------------- # copied from ProviderConfig in PluginManager.get_search_plugins() priority: int - # copied from ProviderConfig in PluginManager.get_search_plugins() - products: Dict[str, Any] # per product type metadata-mapping, set in core._prepare_search product_type_config: Dict[str, Any] @@ -394,8 +394,6 @@ class OrderStatus(TypedDict): #: :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch` #: Maximum number of connections for HTTP requests max_connections: int - #: :class:`~eodag.plugins.search.base.Search` Time to wait until request timeout in seconds - timeout: float #: :class:`~eodag.plugins.search.build_search_result.BuildSearchResult` #: Whether end date should be excluded from search request or not end_date_excluded: bool @@ -415,6 +413,12 @@ class OrderStatus(TypedDict): output_extension: str #: :class:`~eodag.plugins.download.base.Download` Whether the directory structure should be flattened or not flatten_top_dirs: bool + #: :class:`~eodag.plugins.download.base.Download` Level in extracted path tree where to find data + archive_depth: int + #: :class:`~eodag.plugins.download.base.Download` Whether ignore assets and download using ``downloadLink`` or not + ignore_assets: bool + #: :class:`~eodag.plugins.download.base.Download` Product type specific configuration + products: Dict[str, Dict[str, Any]] #: :class:`~eodag.plugins.download.http.HTTPDownload` Whether the product has to be ordered to download it or not order_enabled: bool #: :class:`~eodag.plugins.download.http.HTTPDownload` HTTP request method for the order request @@ -430,6 +434,8 @@ class OrderStatus(TypedDict): #: :class:`~eodag.plugins.download.http.HTTPDownload` #: Do not authenticate the download request but only the order and order status ones no_auth_download: bool + #: :class:`~eodag.plugins.download.http.HTTPDownload` Parameters to be added to the query params of the request + dl_url_params: Dict[str, str] #: :class:`~eodag.plugins.download.s3rest.S3RestDownload` #: At which level of the path part of the url the bucket can be found bucket_path_level: int diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py index 0ddc370c1..36adf1a6b 100644 --- a/eodag/plugins/download/http.py +++ b/eodag/plugins/download/http.py @@ -98,35 +98,50 @@ class HTTPDownload(Download): """HTTPDownload plugin. Handles product download over HTTP protocol - The configuration parameters for this plugin are: - - * **type** [str] (mandatory): HTTPDownload - * **base_uri** [str]: default endpoint url - * **method** [str]: HTTP request method for the download request (GET or POST); default: GET - * **extract** [bool]: if the content of the downloaded file should be extracted; default: True - * **auth_error_code** [int]: which error code is returned in case of an authentication error - * **dl_url_params** [Dict[str, Any]]: parameters to be added to the query params of the request - * **archive_depth** [int]: level in extracted path tree where to find data; default: 1 - * **flatten_top_dirs** [bool]: if the directory structure should be flattened; default: True - * **ignore_assets** [bool]: ignore assets and download using downloadLink; default: False - * **timeout** [int]: time to wait until request timeout in seconds; default: 5 - * **ssl_verify** [bool]: if the ssl certificates should be verified in requests; default: True - * **output_extension** [str]: which extension should be used for the downloaded file - * **no_auth_download** [bool]: if the download should be done without authentication; default: True - * **order_enabled** [bool]: if the product has to be ordered to download it; if this paramter - is set to true, a mapping for the orderLink has to be added to the metadata mapping of - the search plugin used for the provider; default: False - * **order_method** [str]: HTTP request method for the order request (GET or POST); default: GET - * **order_headers** [Dict[str, str]]: headers to be added to the order request - * **order_on_response** [:class:`~eodag.config.PluginConfig.OrderOnResponse`]: a typed dictionary - containing the key 'metadata_mapping' which can be used to add new product properties - based on the data in response to the order request - * **order_status** [:class:`~eodag.config.PluginConfig.OrderStatus`]: configuration to handle - the order status; contains information which method to use, how the response data is interpreted, - which status corresponds to success, ordered and error and what should be done on success. - * **products** [Dict[str, Dict[str, Any]]: product type specific config; the keys are the product types, - the values are dictionaries which can contain the keys output_extension and extract to overwrite - the provider config for a specific product type + :param provider: provider name + :param config: Download plugin configuration: + + * :attr:`~eodag.config.PluginConfig.type` (``str``): (mandatory) ``HTTPDownload`` + * :attr:`~eodag.config.PluginConfig.base_uri` (``str``): default endpoint url + * :attr:`~eodag.config.PluginConfig.method` (``str``): HTTP request method for the download request (``GET`` or + ``POST``); default: ``GET`` + * :attr:`~eodag.config.PluginConfig.extract` (``bool``): if the content of the downloaded file should be + extracted; default: ``True`` + * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an + authentication error + * :attr:`~eodag.config.PluginConfig.dl_url_params` [Dict[str, Any]]: parameters to be added to the query params + of the request + * :attr:`~eodag.config.PluginConfig.archive_depth` (``int``): level in extracted path tree where to find data; + default: ``1`` + * :attr:`~eodag.config.PluginConfig.flatten_top_dirs` (``bool``): if the directory structure should be + flattened; default: ``True`` + * :attr:`~eodag.config.PluginConfig.ignore_assets` (``bool``): ignore assets and download using downloadLink; + default: ``False`` + * :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds; + default: ``5`` + * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in + requests; default: ``True`` + * :attr:`~eodag.config.PluginConfig.output_extension` (``str``): which extension should be used for the + downloaded file + * :attr:`~eodag.config.PluginConfig.no_auth_download` (``bool``): if the download should be done without + authentication; default: ``True`` + * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to be ordered to download it; + if this paramter is set to true, a mapping for the orderLink has to be added to the metadata mapping of + the search plugin used for the provider; default: False + * :attr:`~eodag.config.PluginConfig.order_method` (``str``): HTTP request method for the order request (``GET`` + or ``POST``); default: ``GET`` + * :attr:`~eodag.config.PluginConfig.order_headers` (``[Dict[str, str]]``): headers to be added to the order + request + * :attr:`~eodag.config.PluginConfig.order_on_response` (:class:`~eodag.config.PluginConfig.OrderOnResponse`): + a typed dictionary containing the key ``metadata_mapping`` which can be used to add new product properties + based on the data in response to the order request + * :attr:`~eodag.config.PluginConfig.order_status` (:class:`~eodag.config.PluginConfig.OrderStatus`): + configuration to handle the order status; contains information which method to use, how the response data is + interpreted, which status corresponds to success, ordered and error and what should be done on success. + * :attr:`~eodag.config.PluginConfig.products` (``Dict[str, Dict[str, Any]``): product type specific config; the + keys are the product types, the values are dictionaries which can contain the keys + :attr:`~eodag.config.PluginConfig.output_extension` and :attr:`~eodag.config.PluginConfig.extract` to + overwrite the provider config for a specific product type """ @@ -145,12 +160,13 @@ def order_download( and has `orderLink` in its properties. Product ordering can be configured using the following download plugin parameters: - - **order_enabled**: Wether order is enabled or not (may not use this method + - :attr:`~eodag.config.PluginConfig.order_enabled`: Wether order is enabled or not (may not use this method if no `orderLink` exists) - - **order_method**: (optional) HTTP request method, GET (default) or POST + - :attr:`~eodag.config.PluginConfig.order_method`: (optional) HTTP request method, GET (default) or POST - - **order_on_response**: (optional) things to do with obtained order response: + - :attr:`~eodag.config.PluginConfig.order_on_response`: (optional) things to do with obtained order + response: - *metadata_mapping*: edit or add new product propoerties properties @@ -264,7 +280,7 @@ def order_download_status( It will be executed before each download retry. Product order status request can be configured using the following download plugin parameters: - - **order_status**: :class:`~eodag.config.PluginConfig.OrderStatus` + - :attr:`~eodag.config.PluginConfig.order_status`: :class:`~eodag.config.PluginConfig.OrderStatus` Product properties used for order status: From f2bddd45a721a52362439e68e3a2a96f57961d53 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Fri, 27 Sep 2024 15:21:27 +0200 Subject: [PATCH 10/25] docs: QueryStringSearch params styling --- eodag/config.py | 60 +++++- eodag/plugins/search/qssearch.py | 329 +++++++++++++++++-------------- 2 files changed, 231 insertions(+), 158 deletions(-) diff --git a/eodag/config.py b/eodag/config.py index da3a56bd6..36d53c2df 100644 --- a/eodag/config.py +++ b/eodag/config.py @@ -263,6 +263,41 @@ class DiscoverMetadata(TypedDict): #: Path to the metadata in search result metadata_path: str + class DiscoverProductTypes(TypedDict, total=False): + """Configuration for product types discovery""" + + #: URL from which the product types can be fetched + fetch_url: Optional[str] + #: Type of the provider result + result_type: str + #: JsonPath to the list of product types + results_entry: Union[JSONPath, str] + #: Mapping for the product type id + generic_product_type_id: str + #: Mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed from the provider + #: result + generic_product_type_parsable_metadata: Dict[str, str] + #: Mapping for product type properties which can be parsed from the result that are not product type metadata + generic_product_type_parsable_properties: Dict[str, str] + #: URL to fetch data for a single collection + single_collection_fetch_url: str + #: Query string to be added to the fetch_url to filter for a collection + single_collection_fetch_qs: str + #: Mapping for product type metadata returned by the endpoint given in single_collection_fetch_url + single_product_type_parsable_metadata: Dict[str, str] + + class DiscoverQueryables(TypedDict, total=False): + """Configuration for queryables discovery""" + + #: URL to fetch the queryables valid for all product types + fetch_url: Optional[str] + #: URL to fetch the queryables for a specific product type + product_type_fetch_url: Optional[str] + #: Type of the result + result_type: str + #: JsonPath to retrieve the queryables from the provider result + results_entry: str + class OrderOnResponse(TypedDict): """Configuration for order on-response during download""" @@ -284,7 +319,7 @@ class OrderStatusSuccess(TypedDict): #: Success value for status response HTTP code http_code: int - class OrderStatusOrdered(TypedDict): + class OrderStatusOrdered(TypedDict, total=False): """ Configuration to identify order status ordered during download """ @@ -292,7 +327,7 @@ class OrderStatusOrdered(TypedDict): #: HTTP code of the order status response http_code: int - class OrderStatusRequest(TypedDict): + class OrderStatusRequest(TypedDict, total=False): """ Order status request configuration """ @@ -302,7 +337,7 @@ class OrderStatusRequest(TypedDict): #: Request hearders headers: Dict[str, Any] - class OrderStatusOnSuccess(TypedDict): + class OrderStatusOnSuccess(TypedDict, total=False): """Configuration for order status on-success during download""" #: Whether a new search is needed on success or not @@ -314,7 +349,7 @@ class OrderStatusOnSuccess(TypedDict): #: Metadata-mapping to apply to the success status result metadata_mapping: Dict[str, Union[str, List[str]]] - class OrderStatus(TypedDict): + class OrderStatus(TypedDict, total=False): """Configuration for order status during download""" #: Order status request configuration @@ -365,15 +400,28 @@ class OrderStatus(TypedDict): #: :class:`~eodag.plugins.search.base.Search` Configuration for the metadata auto-discovery discover_metadata: PluginConfig.DiscoverMetadata #: :class:`~eodag.plugins.search.base.Search` Configuration for the product types auto-discovery - discover_product_types: Dict[str, Any] + discover_product_types: DiscoverProductTypes #: :class:`~eodag.plugins.search.base.Search` Configuration for the queryables auto-discovery - discover_queryables: Dict[str, Any] + discover_queryables: DiscoverQueryables #: :class:`~eodag.plugins.search.base.Search` The mapping between eodag metadata and the plugin specific metadata metadata_mapping: Dict[str, Union[str, List[str]]] #: :class:`~eodag.plugins.search.base.Search` URL of the constraint file used to build queryables constraints_file_url: str + #: :class:`~eodag.plugins.search.base.Search` + #: Key which is used in the eodag configuration to map the eodag product type to the provider product type + constraints_file_dataset_key: str + #: :class:`~eodag.plugins.search.base.Search` Key in the json result where the constraints can be found + constraints_entry: str + #: :class:`~eodag.plugins.search.base.Search` + #: Whether only a provider result containing constraints_entry is accepted as valid and used to create constraints + #: or not + stop_without_constraints_entry_key: bool #: :class:`~eodag.plugins.search.base.Search` Parameters to remove from queryables remove_from_queryables: List[str] + #: :class:`~eodag.plugins.search.base.Search` Parameters to be passed as is in the search url query string + literal_search_params: Dict[str, str] + #: :class:`~eodag.plugins.search.qssearch.QueryStringSearch` Characters that should not be quoted in the url params + dont_quote: List[str] #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict describing free text search request build free_text_search_operations: Dict[str, Any] #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict used to simplify further metadata extraction diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index dc23ee627..e24aebc69 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -112,165 +112,190 @@ class QueryStringSearch(Search): """A plugin that helps implementing any kind of search protocol that relies on query strings (e.g: opensearch). Most of the other search plugins inherit from this plugin. - The available configuration parameters for this kind of plugin are: - - * **result_type** [str]: One of "json" or "xml", depending on the - representation of the provider's search results. The default is "json". - * **results_entry** [str] (mandatory): The name of the key in the provider search - result that gives access to the result entries - * **api_endpoint** [str] (mandatory): The endpoint of the provider's search interface - * **need_auth** [bool]: if authentication is needed for the search request; default: False - * **auth_error_code** [int]: which error code is returned in case of an authentication - error; only used if `need_auth=true` - * **ssl_verify** [bool]: if the ssl certificates should be verified in requests; default: True - * **dont_quote** [List[str]]: characters that should not be quoted in the url params - * **timeout** [int]: time to wait until request timeout in seconds; default: 5 - * **literal_search_params** [Dict[str, str]]: A mapping of (search_param => search_value) pairs - giving search parameters to be passed as is in the search url query string. This is useful - for example in situations where the user wants to add a fixed search query parameter exactly - as it is done on the provider interface. - * **pagination** [:class:`~eodag.config.PluginConfig.Pagination`] (mandatory): The configuration of - how the pagination is done on the provider. It is a tree with the following nodes: - - * **next_page_url_tpl** [str] (mandatory): The template for pagination requests. This is a simple - Python format string which will be resolved using the following keywords: - ``url`` (the base url of the search endpoint), ``search`` (the query string corresponding - to the search request), ``items_per_page`` (the number of items to return per page), - ``skip`` (the number of items to skip) or ``skip_base_1`` (the number of items to skip, - starting from 1) and ``page`` (which page to return). - * **total_items_nb_key_path** [str]: An XPath or JsonPath leading to the total number of - results satisfying a request. This is used for providers which provides the total results - metadata along with the result of the query and don't have an endpoint for querying - the number of items satisfying a request, or for providers for which the count endpoint - returns a json or xml document - * **count_endpoint** [str]: The endpoint for counting the number of items satisfying a request - * **count_tpl** [str]: template for the count parameter that should be added to the search request - * **next_page_url_key_path** [str]: A JSONPATH expression used to retrieve - the URL of the next page in the response of the current page. - * **max_items_per_page** [int]: The maximum number of items per page that the provider can - handle; default: 50 - * **start_page** [int]: number of the first page; default: 1 - - * **discover_product_types** [Dict[str, Any]]: configuration for product type discovery - based on information from the provider; It contains the keys: - - * **fetch_url** [str] (mandatory): url from which the product types can be fetched - * **result_type** [str]: type of the provider result; currently only "json" is supported, - (other types could be used in an extension of this plugin) - * **results_entry** [str] (mandatory): json path to the list of product types - * **generic_product_type_id** [str]: mapping for the product type id - * **generic_product_type_parsable_metadata** [Dict[str, str]]: mapping for product type - metadata (e.g. abstract, licence) which can be parsed from the provider result - * **generic_product_type_parsable_properties** [Dict[str, str]]: mapping for product type - properties which can be parsed from the result that are not product type metadata - * **single_collection_fetch_url** [str]: url to fetch data for a single collection; - used if product type metadata is not available from the endpoint given in `fetch_url` - * **single_collection_fetch_qs** [str]: query string to be added to the `fetch_url` - to filter for a collection - * **single_product_type_parsable_metadata** [Dict[str, str]]: mapping for product type - metadata returned by the endpoint given in `single_collection_fetch_url`. - - * **sort** [:class:`~eodag.config.PluginConfig.Sort`]: configuration for sorting the - results. It contains the keys: - - * **sort_by_default** [List[Tuple(str, Literal["ASC", "DESC"])]]: parameter and sort order by - which the result will be sorted by default (if the user does not enter a sort_by parameter); - if not given the result will use the default sorting of the provider; Attention: for some - providers sorting might cause a timeout if no filters are used. In that case no default - sort parameters should be given. The format is:: - - sort_by_default: - - !!python/tuple [, (ASC or DESC)] - - * **sort_by_tpl** [str]: template for the sort parameter that is added to the request; - It contains the parameters `sort_param` and `sort_order` which will be replaced by user input - or default value. If the parameters are added as query params to a GET request, the string - should start with '&', otherwise it should be a valid json string surrounded by '{{ }}'. - * **sort_param_mapping** [Dict [str, str]]: mapping for the parameters available for sorting - * **sort_order_mapping** [Dict[Literal["ascending", "descending"], str]]: mapping for - the sort order - * **max_sort_params** [int]: maximum number of sort parameters supported by the provider; - used to validate the user input to avoid failed requests or unexpected behaviour - (not all parameters are used in the request) - - * **metadata_mapping** [Dict[str, Any]]: The search plugins of this kind can detect when a - metadata mapping is "query-able", and get the semantics of how to format the query string - parameter that enables to make a query on the corresponding metadata. To make a metadata query-able, - just configure it in the metadata mapping to be a list of 2 items, the first one being the - specification of the query string search formatting. The later is a string following the - specification of Python string formatting, with a special behaviour added to it. For example, - an entry in the metadata mapping of this kind:: - - completionTimeFromAscendingNode: - - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}' - - '$.properties.acquisition.endViewingDate' - - means that the search url will have a query string parameter named *"f"* with a value of - *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done with the value - of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that - ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value - of ``completionTimeFromAscendingNode``. This example shows all there is to know about the - semantics of the query string formatting introduced by this plugin: any eodag search parameter - can be referenced in the query string with an additional optional conversion function that - is separated from it by a ``#`` (see :func:`~eodag.utils.format_metadata` for further details - on the available converters). Note that for the values in the ``free_text_search_operations`` - configuration parameter follow the same rule. If the metadata_mapping is not a list but - only a string, this means that the parameters is not queryable but it is included in the - result obtained from the provider. The string indicates how the provider result should be - mapped to the eodag parameter. - * **discover_metadata** [:class:`~eodag.config.PluginConfig.DiscoverMetadata`]: configuration - for the auto-discovery of queryable parameters as well as parameters returned by the provider - which are not in the metadata mapping. It has the attributes: - - * **auto_discovery** [bool]: if the automatic discovery of metadata is activated; default: - False; if false, the other parameters are not used; - * **metadata_pattern** [str]: regex string a parameter in the result should match so - that is used - * **search_param** [Union [str, Dict[str, Any]]]: format to add a query param given by - the user and not in the metadata mapping to the requets, 'metadata' will be replaced by - the search param; can be a string or a dict containing `free_text_search_operations` (see below) - * **metadata_path** [str]: path where the queryable properties can be found in the provider result - - * **free_text_search_operations**: (optional) A tree structure of the form:: - - # noqa: E800 - : # e.g: $search - union: # how to join the operations below (e.g: ' AND ' --> - # '(op1 AND op2) AND (op3 OR op4)') - wrapper: # a pattern for how each operation will be wrapped - # (e.g: '({})' --> '(op1 AND op2)') - operations: # The operations to build - : # e.g: AND - - # e.g: - # 'sensingStartDate:[{startTimeFromAscendingNode}Z TO *]' - - # e.g: - # 'sensingStopDate:[* TO {completionTimeFromAscendingNode}Z]' + :param provider: provider name + :param config: Download plugin configuration: + + * :attr:`~eodag.config.PluginConfig.result_type` (``str``): One of ``json`` or ``xml``, depending on the + representation of the provider's search results. The default is ``json``. + * :attr:`~eodag.config.PluginConfig.results_entry` (``str``) (**mandatory**): The name of the key in the + provider search result that gives access to the result entries + * :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): The endpoint of the provider's + search interface + * :attr:`~eodag.config.PluginConfig.need_auth` (``bool``): if authentication is needed for the search request; + default: ``False`` + * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an + authentication error; only used if ``need_auth=true`` + * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in + requests; default: ``True`` + * :attr:`~eodag.config.PluginConfig.dont_quote` (``List[str]``): characters that should not be quoted in the + url params + * :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds; + default: ``5`` + * :attr:`~eodag.config.PluginConfig.literal_search_params` (``Dict[str, str]``): A mapping of (search_param => + search_value) pairs giving search parameters to be passed as is in the search url query string. This is useful + for example in situations where the user wants to add a fixed search query parameter exactly + as it is done on the provider interface. + * :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`) + (**mandatory**): The configuration of how the pagination is done on the provider. It is a tree with the + following nodes: + + * :attr:`~eodag.config.PluginConfig.Pagination.next_page_url_tpl` (``str``) (**mandatory**): The template for + pagination requests. This is a simple Python format string which will be resolved using the following + keywords: ``url`` (the base url of the search endpoint), ``search`` (the query string corresponding + to the search request), ``items_per_page`` (the number of items to return per page), + ``skip`` (the number of items to skip) or ``skip_base_1`` (the number of items to skip, + starting from 1) and ``page`` (which page to return). + * :attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path` (``str``): An XPath or JsonPath + leading to the total number of results satisfying a request. This is used for providers which provides the + total results metadata along with the result of the query and don't have an endpoint for querying + the number of items satisfying a request, or for providers for which the count endpoint + returns a json or xml document + * :attr:`~eodag.config.PluginConfig.Pagination.count_endpoint` (``str``): The endpoint for counting the number + of items satisfying a request + * :attr:`~eodag.config.PluginConfig.Pagination.count_tpl` (``str``): template for the count parameter that + should be added to the search request + * :attr:`~eodag.config.PluginConfig.Pagination.next_page_url_key_path` (``str``): A JsonPath expression used + to retrieve the URL of the next page in the response of the current page. + * :attr:`~eodag.config.PluginConfig.Pagination.max_items_per_page` (``int``): The maximum number of items per + page that the provider can handle; default: ``50`` + * :attr:`~eodag.config.PluginConfig.Pagination.start_page` (``int``): number of the first page; default: ``1`` + + * :attr:`~eodag.config.PluginConfig.discover_product_types` + (:class:`~eodag.config.PluginConfig.DiscoverProductTypes`): configuration for product type discovery based on + information from the provider; It contains the keys: + + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` (``str``) (**mandatory**): url from which + the product types can be fetched + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.result_type` (``str``): type of the provider result; + currently only ``json`` is supported (other types could be used in an extension of this plugin) + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.results_entry` (``str``) (**mandatory**): json path + to the list of product types + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_id` (``str``): mapping for the + product type id + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_metadata` + (``Dict[str, str]``): mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed + from the provider result + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_properties` + (``Dict[str, str]``): mapping for product type properties which can be parsed from the result that are not + product type metadata + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url` (``str``): url to fetch + data for a single collection; used if product type metadata is not available from the endpoint given in + :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_qs` (``str``): query string + to be added to the :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` to filter for a + collection + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_product_type_parsable_metadata` + (``Dict[str, str]``): mapping for product type metadata returned by the endpoint given in + :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url`. + + * :attr:`~eodag.config.PluginConfig.sort` (:class:`~eodag.config.PluginConfig.Sort`): configuration for sorting + the results. It contains the keys: + + * :attr:`~eodag.config.PluginConfig.Sort.sort_by_default` (``List[Tuple(str, Literal["ASC", "DESC"])]``): + parameter and sort order by which the result will be sorted by default (if the user does not enter a + ``sort_by`` parameter); if not given the result will use the default sorting of the provider; Attention: + for some providers sorting might cause a timeout if no filters are used. In that case no default + sort parameters should be given. The format is:: + + sort_by_default: + - !!python/tuple [, (ASC or DESC)] + + * :attr:`~eodag.config.PluginConfig.Sort.sort_by_tpl` (``str``): template for the sort parameter that is added + to the request; It contains the parameters `sort_param` and `sort_order` which will be replaced by user + input or default value. If the parameters are added as query params to a GET request, the string + should start with ``&``, otherwise it should be a valid json string surrounded by ``{{ }}``. + * :attr:`~eodag.config.PluginConfig.Sort.sort_param_mapping` (``Dict [str, str]``): mapping for the parameters + available for sorting + * :attr:`~eodag.config.PluginConfig.Sort.sort_order_mapping` + (``Dict[Literal["ascending", "descending"], str]``): mapping for the sort order + * :attr:`~eodag.config.PluginConfig.Sort.max_sort_params` (``int``): maximum number of sort parameters + supported by the provider; used to validate the user input to avoid failed requests or unexpected behaviour + (not all parameters are used in the request) + + * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Any]``): The search plugins of this kind can + detect when a metadata mapping is "query-able", and get the semantics of how to format the query string + parameter that enables to make a query on the corresponding metadata. To make a metadata query-able, + just configure it in the metadata mapping to be a list of 2 items, the first one being the + specification of the query string search formatting. The later is a string following the + specification of Python string formatting, with a special behaviour added to it. For example, + an entry in the metadata mapping of this kind:: + + completionTimeFromAscendingNode: + - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}' + - '$.properties.acquisition.endViewingDate' + + means that the search url will have a query string parameter named *"f"* with a value of + *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done with the value + of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that + ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value + of ``completionTimeFromAscendingNode``. This example shows all there is to know about the + semantics of the query string formatting introduced by this plugin: any eodag search parameter + can be referenced in the query string with an additional optional conversion function that + is separated from it by a ``#`` (see :func:`~eodag.api.product.metadata_mapping.format_metadata` for further + details on the available converters). Note that for the values in the ``free_text_search_operations`` + configuration parameter follow the same rule. If the metadata_mapping is not a list but + only a string, this means that the parameters is not queryable but it is included in the + result obtained from the provider. The string indicates how the provider result should be + mapped to the eodag parameter. + * :attr:`~eodag.config.PluginConfig.discover_metadata` (:class:`~eodag.config.PluginConfig.DiscoverMetadata`): + configuration for the auto-discovery of queryable parameters as well as parameters returned by the provider + which are not in the metadata mapping. It has the attributes: + + * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.auto_discovery` (``bool``): if the automatic discovery of + metadata is activated; default: ``False``; if false, the other parameters are not used; + * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.metadata_pattern` (``str``): regex string a parameter in + the result should match so that is used + * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.search_param` (``Union [str, Dict[str, Any]]``): format + to add a query param given by the user and not in the metadata mapping to the requets, 'metadata' will be + replaced by the search param; can be a string or a dict containing `free_text_search_operations` (see below) + * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.metadata_path` (``str``): path where the queryable + properties can be found in the provider result + + * :attr:`~eodag.config.PluginConfig.free_text_search_operations`: (optional) A tree structure of the form:: + + # noqa: E800 + : # e.g: $search + union: # how to join the operations below (e.g: ' AND ' --> + # '(op1 AND op2) AND (op3 OR op4)') + wrapper: # a pattern for how each operation will be wrapped + # (e.g: '({})' --> '(op1 AND op2)') + operations: # The operations to build + : # e.g: AND + - # e.g: + # 'sensingStartDate:[{startTimeFromAscendingNode}Z TO *]' + - # e.g: + # 'sensingStopDate:[* TO {completionTimeFromAscendingNode}Z]' + ... ... - ... - ... + ... With the structure above, each operation will become a string of the form: '( )', then the operations will be joined together using the union string and finally if the number of operations is greater than 1, they will be wrapped as specified by the wrapper config key. - * **discover_queryables** [Dict[str, Any]]: configuration to fetch the queryables from a - provider queryables endpoint; It has the following keys: - - * **fetch_url** [str]: url to fetch the queryables valid for all product types - * **product_type_fetch_url** [str]: url to fetch the queryables for a specific product type - * **result_type** [str]: type of the result (currently only json is used) - * **results_entry** [str]: json path to retrieve the queryables from the provider result - - * **constraints_file_url** [str]: url to fetch the constraints for a specific product type, - can be an http url or a path to a file; the constraints are used to build queryables - * **constraints_file_dataset_key** [str]: key which is used in the eodag configuration to - map the eodag product type to the provider product type; default: dataset - * **constraints_entry** [str]: key in the json result where the constraints can be found; - if not given, it is assumed that the constraints are on top level of the result, i.e. - the result is an array of constraints - * **stop_without_constraints_entry_key** [bool]: if true only a provider result containing - `constraints_entry` is accepted as valid and used to create constraints; default: false + * :attr:`~eodag.config.PluginConfig.discover_queryables` + (:class:`~eodag.config.PluginConfig.DiscoverQueryables`): configuration to fetch the queryables from a + provider queryables endpoint; It has the following keys: + + * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` (``str``): url to fetch the queryables valid + for all product types + * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.product_type_fetch_url` (``str``): url to fetch the + queryables for a specific product type + * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.result_type` (``str``): type of the result (currently + only ``json`` is used) + * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.results_entry` (``str``): json path to retrieve the + queryables from the provider result + + * :attr:`~eodag.config.PluginConfig.constraints_file_url` (``str``): url to fetch the constraints for a specific + product type, can be an http url or a path to a file; the constraints are used to build queryables + * :attr:`~eodag.config.PluginConfig.constraints_file_dataset_key` (``str``): key which is used in the eodag + configuration to map the eodag product type to the provider product type; default: ``dataset`` + * :attr:`~eodag.config.PluginConfig.constraints_entry` (``str``): key in the json result where the constraints + can be found; if not given, it is assumed that the constraints are on top level of the result, i.e. + the result is an array of constraints + * :attr:`~eodag.config.PluginConfig.stop_without_constraints_entry_key` (``bool``): if true only a provider + result containing `constraints_entry` is accepted as valid and used to create constraints; default: ``False`` """ From 008e6dfe31bebb8b8e070dc9772c280a3d6c5b67 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Fri, 27 Sep 2024 15:28:34 +0200 Subject: [PATCH 11/25] refactor: QueryStringSearch params styling --- eodag/plugins/search/cop_marine.py | 8 ++--- eodag/plugins/search/qssearch.py | 38 +++++++++++++--------- eodag/plugins/search/static_stac_search.py | 15 ++++----- 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/eodag/plugins/search/cop_marine.py b/eodag/plugins/search/cop_marine.py index cec633726..a37c9493a 100644 --- a/eodag/plugins/search/cop_marine.py +++ b/eodag/plugins/search/cop_marine.py @@ -131,12 +131,10 @@ def _get_product_type_info( ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]: """Fetch product type and associated datasets info""" - fetch_url = cast( - str, - self.config.discover_product_types["fetch_url"].format( - **self.config.__dict__ - ), + fetch_url = cast(str, self.config.discover_product_types["fetch_url"]).format( + **self.config.__dict__ ) + logger.debug("fetch data for collection %s", product_type) provider_product_type = self.config.products.get(product_type, {}).get( "productType", None diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index e24aebc69..5ea1d92cf 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -464,12 +464,10 @@ def discover_product_types(self, **kwargs: Any) -> Optional[Dict[str, Any]]: try: prep = PreparedSearch() - prep.url = cast( - str, - self.config.discover_product_types["fetch_url"].format( - **self.config.__dict__ - ), - ) + fetch_url = self.config.discover_product_types.get("fetch_url") + if fetch_url is None: + return None + prep.url = fetch_url.format(**self.config.__dict__) # get auth if available if "auth" in kwargs: @@ -511,12 +509,14 @@ def discover_product_types(self, **kwargs: Any) -> Optional[Dict[str, Any]]: if self.config.discover_product_types["result_type"] == "json": resp_as_json = response.json() # extract results from response json - result = [ - match.value - for match in self.config.discover_product_types[ - "results_entry" - ].find(resp_as_json) - ] + results_entry = self.config.discover_product_types["results_entry"] + if not isinstance(results_entry, JSONPath): + logger.warning( + f"Could not parse {self.provider} discover_product_types.results_entry" + f" as JSONPath: {results_entry}" + ) + return None + result = [match.value for match in results_entry.find(resp_as_json)] if result and isinstance(result[0], list): result = result[0] @@ -1875,6 +1875,8 @@ def discover_queryables( if provider_product_type else self.config.discover_queryables["fetch_url"] ) + if unparsed_fetch_url is None: + return None fetch_url = unparsed_fetch_url.format( provider_product_type=provider_product_type, **self.config.__dict__ @@ -1902,11 +1904,15 @@ def discover_queryables( resp_as_json = response.json() # extract results from response json - json_queryables = [ - match.value - for match in self.config.discover_queryables["results_entry"].find( - resp_as_json + results_entry = self.config.discover_queryables["results_entry"] + if not isinstance(results_entry, JSONPath): + logger.warning( + f"Could not parse {self.provider} discover_queryables.results_entry" + f" as JSONPath: {results_entry}" ) + return None + json_queryables = [ + match.value for match in results_entry.find(resp_as_json) ][0] except KeyError as e: diff --git a/eodag/plugins/search/static_stac_search.py b/eodag/plugins/search/static_stac_search.py index e24cc52c9..f7a500ec5 100644 --- a/eodag/plugins/search/static_stac_search.py +++ b/eodag/plugins/search/static_stac_search.py @@ -18,7 +18,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from unittest import mock import geojson @@ -82,19 +82,18 @@ def __init__(self, provider: str, config: PluginConfig) -> None: getattr(self.config, "discover_product_types", {}).get("fetch_url") == "{api_endpoint}/../collections" ): - self.config.discover_product_types = {"fetch_url": None} + self.config.discover_product_types = {} def discover_product_types(self, **kwargs: Any) -> Optional[Dict[str, Any]]: """Fetch product types list from a static STAC Catalog provider using `discover_product_types` conf :returns: configuration dict containing fetched product types information """ - fetch_url = cast( - str, - self.config.discover_product_types["fetch_url"].format( - **self.config.__dict__ - ), - ) + unformatted_fetch_url = self.config.discover_product_types.get("fetch_url") + if unformatted_fetch_url is None: + return None + fetch_url = unformatted_fetch_url.format(**self.config.__dict__) + collections = fetch_stac_collections( fetch_url, collection=kwargs.get("q"), From fb0674faf469d5643fe6adc94c4f6586f24323bf Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Fri, 27 Sep 2024 16:31:45 +0200 Subject: [PATCH 12/25] docs: ODataV4Search params styling --- eodag/config.py | 15 ++++- eodag/plugins/search/qssearch.py | 94 +++++++++++++++++--------------- 2 files changed, 64 insertions(+), 45 deletions(-) diff --git a/eodag/config.py b/eodag/config.py index 36d53c2df..950114523 100644 --- a/eodag/config.py +++ b/eodag/config.py @@ -365,6 +365,16 @@ class OrderStatus(TypedDict, total=False): #: Configuration for order status on-success during download on_success: PluginConfig.OrderStatusOnSuccess + class MetadataPreMapping(TypedDict, total=False): + """Configuration which can be used to simplify further metadata extraction""" + + #: JsonPath of the metadata entry + metadata_path: str + #: Key to get the metadata id + metadata_path_id: str + #: Key to get the metadata value + metadata_path_value: str + #: :class:`~eodag.plugins.base.PluginTopic` The name of the plugin class to use to instantiate the plugin object name: str #: :class:`~eodag.plugins.base.PluginTopic` Plugin type @@ -424,8 +434,11 @@ class OrderStatus(TypedDict, total=False): dont_quote: List[str] #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict describing free text search request build free_text_search_operations: Dict[str, Any] + #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Set to ``True`` if the metadata is not given in the search + #: result and a two step search has to be performed + per_product_metadata_query: bool #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict used to simplify further metadata extraction - metadata_pre_mapping: Dict[str, Any] + metadata_pre_mapping: MetadataPreMapping #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` URL to which the data request shall be sent data_request_url: str #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` URL to fetch the status of the data request diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index 5ea1d92cf..bbbe47880 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -113,7 +113,7 @@ class QueryStringSearch(Search): query strings (e.g: opensearch). Most of the other search plugins inherit from this plugin. :param provider: provider name - :param config: Download plugin configuration: + :param config: Search plugin configuration: * :attr:`~eodag.config.PluginConfig.result_type` (``str``): One of ``json`` or ``xml``, depending on the representation of the provider's search results. The default is ``json``. @@ -232,11 +232,11 @@ class QueryStringSearch(Search): semantics of the query string formatting introduced by this plugin: any eodag search parameter can be referenced in the query string with an additional optional conversion function that is separated from it by a ``#`` (see :func:`~eodag.api.product.metadata_mapping.format_metadata` for further - details on the available converters). Note that for the values in the ``free_text_search_operations`` - configuration parameter follow the same rule. If the metadata_mapping is not a list but - only a string, this means that the parameters is not queryable but it is included in the - result obtained from the provider. The string indicates how the provider result should be - mapped to the eodag parameter. + details on the available converters). Note that for the values in the + :attr:`~eodag.config.PluginConfig.free_text_search_operations` configuration parameter follow the same rule. + If the metadata_mapping is not a list but only a string, this means that the parameters is not queryable but + it is included in the result obtained from the provider. The string indicates how the provider result should + be mapped to the eodag parameter. * :attr:`~eodag.config.PluginConfig.discover_metadata` (:class:`~eodag.config.PluginConfig.DiscoverMetadata`): configuration for the auto-discovery of queryable parameters as well as parameters returned by the provider which are not in the metadata mapping. It has the attributes: @@ -247,33 +247,12 @@ class QueryStringSearch(Search): the result should match so that is used * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.search_param` (``Union [str, Dict[str, Any]]``): format to add a query param given by the user and not in the metadata mapping to the requets, 'metadata' will be - replaced by the search param; can be a string or a dict containing `free_text_search_operations` (see below) + replaced by the search param; can be a string or a dict containing + :attr:`~eodag.config.PluginConfig.free_text_search_operations` + (see :class:`~eodag.plugins.search.qssearch.ODataV4Search`) * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.metadata_path` (``str``): path where the queryable properties can be found in the provider result - * :attr:`~eodag.config.PluginConfig.free_text_search_operations`: (optional) A tree structure of the form:: - - # noqa: E800 - : # e.g: $search - union: # how to join the operations below (e.g: ' AND ' --> - # '(op1 AND op2) AND (op3 OR op4)') - wrapper: # a pattern for how each operation will be wrapped - # (e.g: '({})' --> '(op1 AND op2)') - operations: # The operations to build - : # e.g: AND - - # e.g: - # 'sensingStartDate:[{startTimeFromAscendingNode}Z TO *]' - - # e.g: - # 'sensingStopDate:[* TO {completionTimeFromAscendingNode}Z]' - ... - ... - ... - - With the structure above, each operation will become a string of the form: - '( )', then the operations will be joined together using - the union string and finally if the number of operations is greater than 1, - they will be wrapped as specified by the wrapper config key. - * :attr:`~eodag.config.PluginConfig.discover_queryables` (:class:`~eodag.config.PluginConfig.DiscoverQueryables`): configuration to fetch the queryables from a provider queryables endpoint; It has the following keys: @@ -296,7 +275,6 @@ class QueryStringSearch(Search): the result is an array of constraints * :attr:`~eodag.config.PluginConfig.stop_without_constraints_entry_key` (``bool``): if true only a provider result containing `constraints_entry` is accepted as valid and used to create constraints; default: ``False`` - """ extract_properties: Dict[str, Callable[..., Dict[str, Any]]] = { @@ -1277,19 +1255,47 @@ def _request( class ODataV4Search(QueryStringSearch): - """A specialisation of a QueryStringSearch that does a two step search to retrieve - all products metadata. All configuration parameters of QueryStringSearch are also available - for this plugin. In addition, the following parameters can be configured: - - * **per_product_metadata_query** [bool]: should be set to true if the metadata is not given - in the search result and a two step search has to be performed; default: false - * **metadata_pre_mapping** [Dict[str, str]]: a dictionary which can be used to simplify - further metadata extraction. For example, going from '$.Metadata[?(@.id="foo")].value' - to '$.Metadata.foo.value'. It has the keys: - - * **metadata_path** [str]: json path of the metadata entry - * **metadata_path_id** [str]: key to get the metadata id - * **metadata_path_value** [str]: key to get the metadata value + """A specialisation of a :class:`~eodag.plugins.search.qssearch.QueryStringSearch` that does a two step search to + retrieve all products metadata. All configuration parameters of + :class:`~eodag.plugins.search.qssearch.QueryStringSearch` are also available for this plugin. In addition, the + following parameters can be configured: + + :param provider: provider name + :param config: Search plugin configuration: + + * :attr:`~eodag.config.PluginConfig.per_product_metadata_query` (``bool``): should be set to true if the metadata + is not given in the search result and a two step search has to be performed; default: false + * :attr:`~eodag.config.PluginConfig.metadata_pre_mapping` (:class:`~eodag.config.PluginConfig.MetadataPreMapping`) + : a dictionary which can be used to simplify further metadata extraction. For example, going from + ``$.Metadata[?(@.id="foo")].value`` to ``$.Metadata.foo.value``. It has the keys: + + * :attr:`~eodag.config.PluginConfig.MetadataPreMapping.metadata_path` (``str``): json path of the metadata entry + * :attr:`~eodag.config.PluginConfig.MetadataPreMapping.metadata_path_id` (``str``): key to get the metadata id + * :attr:`~eodag.config.PluginConfig.MetadataPreMapping.metadata_path_value` (``str``): key to get the metadata + value + + * :attr:`~eodag.config.PluginConfig.free_text_search_operations`: (optional) A tree structure of the form:: + + # noqa: E800 + : # e.g: $search + union: # how to join the operations below (e.g: ' AND ' --> + # '(op1 AND op2) AND (op3 OR op4)') + wrapper: # a pattern for how each operation will be wrapped + # (e.g: '({})' --> '(op1 AND op2)') + operations: # The operations to build + : # e.g: AND + - # e.g: + # 'sensingStartDate:[{startTimeFromAscendingNode}Z TO *]' + - # e.g: + # 'sensingStopDate:[* TO {completionTimeFromAscendingNode}Z]' + ... + ... + ... + + With the structure above, each operation will become a string of the form: + ``( )``, then the operations will be joined together using + the union string and finally if the number of operations is greater than 1, + they will be wrapped as specified by the wrapper config key. """ From 3fa0b0b4d43c10db467e2cbc10a8bcab879c3543 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Fri, 27 Sep 2024 17:33:08 +0200 Subject: [PATCH 13/25] docs: remove misc.copy_overwrite warning --- docs/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index d377729e0..a63acbe1a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -251,6 +251,8 @@ "shapely": ("https://shapely.readthedocs.io/en/stable/", None), } +suppress_warnings = ["misc.copy_overwrite"] + def _build_finished(app, exception): """Post-build pages edit""" From 669a2df47436f5705f82fe73a47f0018b4e5d481 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Fri, 27 Sep 2024 17:52:19 +0200 Subject: [PATCH 14/25] fix: missing TypedDict parent class --- eodag/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eodag/config.py b/eodag/config.py index 950114523..b2aaa7cbc 100644 --- a/eodag/config.py +++ b/eodag/config.py @@ -410,9 +410,9 @@ class MetadataPreMapping(TypedDict, total=False): #: :class:`~eodag.plugins.search.base.Search` Configuration for the metadata auto-discovery discover_metadata: PluginConfig.DiscoverMetadata #: :class:`~eodag.plugins.search.base.Search` Configuration for the product types auto-discovery - discover_product_types: DiscoverProductTypes + discover_product_types: PluginConfig.DiscoverProductTypes #: :class:`~eodag.plugins.search.base.Search` Configuration for the queryables auto-discovery - discover_queryables: DiscoverQueryables + discover_queryables: PluginConfig.DiscoverQueryables #: :class:`~eodag.plugins.search.base.Search` The mapping between eodag metadata and the plugin specific metadata metadata_mapping: Dict[str, Union[str, List[str]]] #: :class:`~eodag.plugins.search.base.Search` URL of the constraint file used to build queryables @@ -438,7 +438,7 @@ class MetadataPreMapping(TypedDict, total=False): #: result and a two step search has to be performed per_product_metadata_query: bool #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict used to simplify further metadata extraction - metadata_pre_mapping: MetadataPreMapping + metadata_pre_mapping: PluginConfig.MetadataPreMapping #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` URL to which the data request shall be sent data_request_url: str #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` URL to fetch the status of the data request From ce696be0af4ff93a0f37342090dcd4c3d2af4afa Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Tue, 1 Oct 2024 17:56:17 +0200 Subject: [PATCH 15/25] docs: use sphinx style for plugin config doc --- eodag/plugins/apis/ecmwf.py | 19 +- eodag/plugins/apis/usgs.py | 33 ++-- eodag/plugins/authentication/aws_auth.py | 9 +- eodag/plugins/authentication/generic.py | 11 +- eodag/plugins/authentication/header.py | 9 +- eodag/plugins/authentication/keycloak.py | 31 ++-- eodag/plugins/authentication/oauth.py | 8 +- .../plugins/authentication/openid_connect.py | 98 +++++----- eodag/plugins/authentication/qsauth.py | 9 +- eodag/plugins/authentication/sas_auth.py | 17 +- eodag/plugins/authentication/token.py | 33 ++-- .../plugins/authentication/token_exchange.py | 24 ++- eodag/plugins/crunch/filter_date.py | 6 +- .../plugins/crunch/filter_latest_tpl_name.py | 4 +- eodag/plugins/crunch/filter_overlap.py | 17 +- eodag/plugins/crunch/filter_property.py | 8 +- eodag/plugins/download/aws.py | 42 +++-- eodag/plugins/download/creodias_s3.py | 13 +- eodag/plugins/download/http.py | 10 +- eodag/plugins/download/s3rest.py | 29 +-- eodag/plugins/search/build_search_result.py | 18 +- eodag/plugins/search/cop_marine.py | 6 +- eodag/plugins/search/creodias_s3.py | 5 +- eodag/plugins/search/csw.py | 64 +++---- eodag/plugins/search/data_request_search.py | 168 ++++++++++-------- eodag/plugins/search/qssearch.py | 32 ++-- eodag/plugins/search/static_stac_search.py | 13 +- 27 files changed, 427 insertions(+), 309 deletions(-) diff --git a/eodag/plugins/apis/ecmwf.py b/eodag/plugins/apis/ecmwf.py index 76d1d1567..9367aa0bf 100644 --- a/eodag/plugins/apis/ecmwf.py +++ b/eodag/plugins/apis/ecmwf.py @@ -72,15 +72,16 @@ class EcmwfApi(Api, BuildPostSearchResult): :class:`~eodag.plugins.search.build_search_result.BuildPostSearchResult` for the creation of the search result. - The configuration parameters for this plugin are: - - * **type** [str] (mandatory): EcmwfApi - * **api_endpoint** [str] (mandatory): url of the ecmwf api - * **metadata_mapping** [Dict[str, Union[str, list]]]: how parameters should be mapped between - the provider and eodag; If a string is given, this is the mapping parameter returned by - provider -> eodag parameter. If a list with 2 elements is given, the first one is the mapping - eodag parameter -> provider query parameters and the second one the mapping provider result - parameter -> eodag parameter + :param provider: provider name + :param config: Api plugin configuration: + + * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): EcmwfApi + * :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): url of the ecmwf api + * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Union[str, list]]``): how + parameters should be mapped between the provider and eodag; If a string is given, this is + the mapping parameter returned by provider -> eodag parameter. If a list with 2 elements + is given, the first one is the mapping eodag parameter -> provider query parameters + and the second one the mapping provider result parameter -> eodag parameter """ def __init__(self, provider: str, config: PluginConfig) -> None: diff --git a/eodag/plugins/apis/usgs.py b/eodag/plugins/apis/usgs.py index d981ec017..b869ff3c3 100644 --- a/eodag/plugins/apis/usgs.py +++ b/eodag/plugins/apis/usgs.py @@ -70,19 +70,26 @@ class UsgsApi(Api): """A plugin that enables to query and download data on the USGS catalogues - The configuration parameters for this plugin are: - - * **type** [str] (mandatory): UsgsApi - * **pagination** [Dict[str, Any]] (mandatory): dict containing parameters for pagination; should contain the - key total_items_nb_key_path which is indicating the key for the number of total items in the provider result - * **ssl_verify** [bool]: if the ssl certificates should be verified in the download request; default: True - * **need_auth** [bool]: if authentication is required for search; default: False - * **extract** [bool]: if the content of the downloaded file should be extracted; default: True - * **order_enabled** [bool]: if the product has to be ordered to download it; default: False - * **metadata_mapping** [Dict[str, Union[str, list]]]: how parameters should be mapped between the provider and eodag - If a string is given, this is the mapping parameter returned by provider -> eodag parameter. If a list with - 2 elements is given, the first one is the mapping eodag parameter -> provider query parameters and the second one - the mapping provider result parameter -> eodag parameter + :param provider: provider name + :param config: Api plugin configuration: + + * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): UsgsApi + * :attr:`~eodag.config.PluginConfig.pagination` (``Dict[str, Any]``) (**mandatory**): dict + containing parameters for pagination; should contain the key ``total_items_nb_key_path`` + which is indicating the key for the number of total items in the provider result + * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates + should be verified in the download request; default: ``True`` + * :attr:`~eodag.config.PluginConfig.need_auth` (``bool``): if authentication is required + for search; default: ``False`` + * :attr:`~eodag.config.PluginConfig.extract` (``bool``): if the content of the downloaded + file should be extracted; default: ``True`` + * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to + be ordered to download it; default: ``False`` + * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Union[str, list]]``): how + parameters should be mapped between the provider and eodag; If a string is given, this is + the mapping parameter returned by provider -> eodag parameter. If a list with 2 elements + is given, the first one is the mapping eodag parameter -> provider query parameters + and the second one the mapping provider result parameter -> eodag parameter """ def __init__(self, provider: str, config: PluginConfig) -> None: diff --git a/eodag/plugins/authentication/aws_auth.py b/eodag/plugins/authentication/aws_auth.py index 02f52abf5..df84f436a 100644 --- a/eodag/plugins/authentication/aws_auth.py +++ b/eodag/plugins/authentication/aws_auth.py @@ -40,11 +40,12 @@ class AwsAuth(Authentication): * auth using current environment (AWS environment variables and/or ``~/aws/*``), will be skipped if AWS credentials are filled in eodag conf - The configuration parameters for this plugin are: + :param provider: provider name + :param config: Authentication plugin configuration: - * **type** [str] (mandatory): AwsAuth - * **auth_error_code** [int] (mandatory for creodias_s3): which error code is returned - in case of an authentication error + * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): AwsAuth + * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): (mandatory for creodias_s3) + which error code is returned in case of an authentication error """ diff --git a/eodag/plugins/authentication/generic.py b/eodag/plugins/authentication/generic.py index b1da2f912..6799f9964 100644 --- a/eodag/plugins/authentication/generic.py +++ b/eodag/plugins/authentication/generic.py @@ -30,14 +30,15 @@ class GenericAuth(Authentication): """GenericAuth authentication plugin (authentication using username and password) + The mandatory parameters that have to be added in the eodag config are username and password. - The configuration parameters for this plugin are: + :param provider: provider name + :param config: Authentication plugin configuration: - * **type** [str] (mandatory): GenericAuth - * **method** [str]: specifies if digest authentication (digest) or basic authentication - (basic) should be used; default: basic + * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): GenericAuth + * :attr:`~eodag.config.PluginConfig.method` (``str``): specifies if digest authentication + (``digest``) or basic authentication (``basic``) should be used; default: ``basic`` - The mandatory parameters that have to be added in the eodag config are username and password. """ def authenticate(self) -> AuthBase: diff --git a/eodag/plugins/authentication/header.py b/eodag/plugins/authentication/header.py index 60bcf77eb..164838d30 100644 --- a/eodag/plugins/authentication/header.py +++ b/eodag/plugins/authentication/header.py @@ -34,10 +34,12 @@ class HTTPHeaderAuth(Authentication): This plugin enables implementation of custom HTTP authentication scheme (other than Basic, Digest, Token negotiation et al.) using HTTP headers. - The configuration parameters for this plugin are: + :param provider: provider name + :param config: Authentication plugin configuration: - * **type** [str] (mandatory): HTTPHeaderAuth - * **headers** [Dict[str, str]]: dictionary containing all keys/value pairs that should be added to the headers + * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): HTTPHeaderAuth + * :attr:`~eodag.config.PluginConfig.headers` (``Dict[str, str]``): dictionary containing + all keys/value pairs that should be added to the headers Below an example for the configuration in the providers config file is shown:: @@ -77,6 +79,7 @@ class HTTPHeaderAuth(Authentication): X-Another-Special-Header: "YYY" ... ... + """ def authenticate(self) -> HeaderAuth: diff --git a/eodag/plugins/authentication/keycloak.py b/eodag/plugins/authentication/keycloak.py index ab7d4b448..a8ec0b172 100644 --- a/eodag/plugins/authentication/keycloak.py +++ b/eodag/plugins/authentication/keycloak.py @@ -42,18 +42,25 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase): """Authentication plugin using Keycloak and OpenId Connect. This plugin requests a token which is added to a query-string or a header for authentication. - The configuration parameters for this plugin are: - - * **type** [str] (mandatory): KeycloakOIDCPasswordAuth - * **auth_base_uri** [str] (mandatory): base url used in the request to fetch the token - * **realm** [str] (mandatory): keycloak realm - * **client_id** [str] (mandatory): keycloak client id - * **client_secret** [str] (mandatory): keycloak client secret, set to null if no secret is used - * **token_provision** [str] (mandatory): if the token should be added to the query string (qs) - or to the header (header) - * **token_qs_key** [str] (mandatory if token_provision=qs): key of the param added to the query string - * **auth_error_code** [int]: which error code is returned in case of an authentication error - * **ssl_verify** [bool]: if the ssl certificates should be verified in the token request; default: True + + :param provider: provider name + :param config: Authentication plugin configuration: + + * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): KeycloakOIDCPasswordAuth + * :attr:`~eodag.config.PluginConfig.auth_base_uri` (``str``) (**mandatory**): base url + used in the request to fetch the token + * :attr:`~eodag.config.PluginConfig.realm` (``str``) (**mandatory**): keycloak realm + * :attr:`~eodag.config.PluginConfig.client_id` (``str``) (**mandatory**): keycloak client id + * :attr:`~eodag.config.PluginConfig.client_secret` (``str``) (**mandatory**): keycloak + client secret, set to null if no secret is used + * :attr:`~eodag.config.PluginConfig.token_provision` (``str``) (**mandatory**): if the + token should be added to the query string (``qs``) or to the header (``header``) + * :attr:`~eodag.config.PluginConfig.token_qs_key` (``str``): (mandatory if token_provision=qs) + key of the param added to the query string + * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is + returned in case of an authentication error + * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates + should be verified in the token request; default: ``True`` Using :class:`~eodag.plugins.download.http.HTTPDownload` a download link `http://example.com?foo=bar` will become diff --git a/eodag/plugins/authentication/oauth.py b/eodag/plugins/authentication/oauth.py index 24ddea304..6c0b2b6b4 100644 --- a/eodag/plugins/authentication/oauth.py +++ b/eodag/plugins/authentication/oauth.py @@ -28,11 +28,13 @@ class OAuth(Authentication): """OAuth authentication plugin - The configuration parameters for this plugin are: + The mandatory parameters that have to be added in the eodag config are aws_access_key_id and aws_secret_access_key. - * **type** [str] (mandatory): OAuth + :param provider: provider name + :param config: Authentication plugin configuration: + + * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): OAuth - The mandatory parameters that have to be added in the eodag config are aws_access_key_id and aws_secret_access_key. """ def __init__(self, provider: str, config: PluginConfig) -> None: diff --git a/eodag/plugins/authentication/openid_connect.py b/eodag/plugins/authentication/openid_connect.py index 4ffce2156..b133dbc14 100644 --- a/eodag/plugins/authentication/openid_connect.py +++ b/eodag/plugins/authentication/openid_connect.py @@ -183,53 +183,61 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase): The headless interaction is fully configurable, and rely on XPATH to retrieve all the necessary information. - The configuration keys of this plugin are as follows (they have no defaults): - - * **type** [str] (mandatory): OIDCAuthorizationCodeFlowAuth - * **authorization_uri** [str] (mandatory): The authorization url of the server (where to query for grants) - * **redirect_uri** [str] (mandatory): The callback url that will handle the code given by the OIDC provider - * **token_uri** [str] (mandatory): The url to query to exchange the authorization - code obtained from the OIDC provider for an authorized token - * **client_id** [str] (mandatory): The OIDC provider's client ID of the eodag provider - * **user_consent_needed** [bool] (mandatory): Whether a user consent is needed - during the authentication - * **token_exchange_post_data_method** [str] (mandatory): One of: json, data or params. - This is the way to pass the data to the POST request that is made to the token server. - They correspond to the recognised keywords arguments of the Python - `requests `_ library - * **token_key** [str] (mandatory): The key pointing to the token in the json response - to the POST request to the token server - * **token_provision** [str] (mandatory): One of qs or header. This is how the token - obtained will be used to authenticate the user on protected requests. If 'qs' is - chosen, then 'token_qs_key' is mandatory - * **login_form_xpath** [str] (mandatory): The xpath to the HTML form element representing - the user login form - * **authentication_uri_source** [str] (mandatory): Where to look for the authentication_uri. - One of 'config' (in the configuration) or 'login-form' (use the 'action' URL found in the - login form retrieved with login_form_xpath). If the value is 'config', - authentication_uri config param is mandatory - * **authentication_uri** [str] (mandatory if authentication_uri_source=config): - The URL of the authentication backend of the OIDC provider - * **user_consent_form_xpath** [str]: The xpath to the user consent form. The form - is searched in the content of the response to the authorization request - * **user_consent_form_data** [Dict[str, str]]: The data that will be passed with the - POST request on the form 'action' URL. The data are given as a key value pairs, the - keys representing the data key and the value being either a 'constant' string value, - or a string of the form 'xpath()' and representing a + :param provider: provider name + :param config: Authentication plugin configuration: + + * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): OIDCAuthorizationCodeFlowAuth + * :attr:`~eodag.config.PluginConfig.authorization_uri` (``str``) (**mandatory**): The + authorization url of the server (where to query for grants) + * :attr:`~eodag.config.PluginConfig.redirect_uri` (``str``) (**mandatory**): The callback + url that will handle the code given by the OIDC provider + * :attr:`~eodag.config.PluginConfig.token_uri` (``str``) (**mandatory**): The url to query + to exchange the authorization code obtained from the OIDC provider for an authorized token + * :attr:`~eodag.config.PluginConfig.client_id` (``str``) (**mandatory**): The OIDC provider's + client ID of the eodag provider + * :attr:`~eodag.config.PluginConfig.user_consent_needed` (``bool`) (mandatory): Whether + a user consent is needed during the authentication + * :attr:`~eodag.config.PluginConfig.token_exchange_post_data_method` (``str``) (**mandatory**): + One of: ``json``, ``data`` or ``params``. This is the way to pass the data to the + POST request that is made to the token server. They correspond to the recognised keywords + arguments of the Python `requests `_ library + * :attr:`~eodag.config.PluginConfig.token_key` (``str``) (**mandatory**): The key pointing + to the token in the json response to the POST request to the token server + * :attr:`~eodag.config.PluginConfig.token_provision` (``str``) (**mandatory**): One of + ``qs`` or ``header``. This is how the token obtained will be used to authenticate the + user on protected requests. If ``qs`` is chosen, then ``token_qs_key`` is mandatory + * :attr:`~eodag.config.PluginConfig.login_form_xpath` (``str``) (**mandatory**): The + xpath to the HTML form element representing the user login form + * :attr:`~eodag.config.PluginConfig.authentication_uri_source` (``str``) (**mandatory**): Where + to look for the authentication_uri. One of ``config`` (in the configuration) or ``login-form`` + (use the 'action' URL found in the login form retrieved with login_form_xpath). If the + value is ``config``, authentication_uri config param is mandatory + * :attr:`~eodag.config.PluginConfig.authentication_uri` (``str``): (mandatory if + authentication_uri_source=config) The URL of the authentication backend of the OIDC provider + * :attr:`~eodag.config.PluginConfig.user_consent_form_xpath` (``str``): The xpath to + the user consent form. The form is searched in the content of the response to the authorization request + * :attr:`~eodag.config.PluginConfig.user_consent_form_data` (``Dict[str, str]``): The data that + will be passed with the POST request on the form 'action' URL. The data are given as + key value pairs, the keys representing the data key and the value being either a 'constant' + string value, or a string of the form 'xpath()' and representing a value to be retrieved in the user consent form. The xpath must resolve directly to a string value, not to an HTML element. Example: `xpath(//input[@name="sessionDataKeyConsent"]/@value)` - * **additional_login_form_data** [Dict[str, str]]: A mapping giving additional data - to be passed to the login POST request. The value follows the same rules as with user_consent_form_data - * **exchange_url_error_pattern** [Dict[str, str]]: Key/value pairs of patterns/messages. - If exchange_url contains the given pattern, the associated message will be sent in an AuthenticationError - * **client_secret** [str]:The OIDC provider's client secret of the eodag provider - * **token_exchange_params** [Dict[str, str]: mandatory keys for the dict: redirect_uri, client_id; - A mapping between OIDC url query string and token handler query string params - (only necessary if they are not the same as for OIDC). This is eodag provider dependant - * **token_qs_key** [str] (mandatory when token_provision=qs): Refers to the name - of the query param to be used in the query request - * **refresh_token_key** [str]: The key pointing to the refresh_token in the json response - to the POST request to the token server + * :attr:`~eodag.config.PluginConfig.additional_login_form_data` (``Dict[str, str]``): A mapping + giving additional data to be passed to the login POST request. The value follows + the same rules as with user_consent_form_data + * :attr:`~eodag.config.PluginConfig.exchange_url_error_pattern` (``Dict[str, str]``): Key/value + pairs of patterns/messages. If exchange_url contains the given pattern, the associated + message will be sent in an AuthenticationError + * :attr:`~eodag.config.PluginConfig.client_secret` (``str``): The OIDC provider's client + secret of the eodag provider + * :attr:`~eodag.config.PluginConfig.token_exchange_params` (``Dict[str, str]``): mandatory + keys for the dict: redirect_uri, client_id; A mapping between OIDC url query string + and token handler query string params (only necessary if they are not the same as for OIDC). + This is eodag provider dependant + * :attr:`~eodag.config.PluginConfig.token_qs_key` (``str``): (mandatory when token_provision=qs) + Refers to the name of the query param to be used in the query request + * :attr:`~eodag.config.PluginConfig.refresh_token_key` (``str``): The key pointing to + the refresh_token in the json response to the POST request to the token server """ SCOPE = "openid" diff --git a/eodag/plugins/authentication/qsauth.py b/eodag/plugins/authentication/qsauth.py index 6e532db8b..ab7513263 100644 --- a/eodag/plugins/authentication/qsauth.py +++ b/eodag/plugins/authentication/qsauth.py @@ -36,10 +36,13 @@ class HttpQueryStringAuth(Authentication): """An Authentication plugin using HTTP query string parameters. This plugin sends credentials as query-string parameters. - The configuration parameters for this plugin are: - * **type** [str] (mandatory): HttpQueryStringAuth - * **auth_uri** [str]: used to check the credentials given in the configuration + :param provider: provider name + :param config: Authentication plugin configuration: + + * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): HttpQueryStringAuth + * :attr:`~eodag.config.PluginConfig.auth_uri` (``str``): used to check the credentials + given in the configuration Using :class:`~eodag.plugins.download.http.HTTPDownload` a download link `http://example.com?foo=bar` will become diff --git a/eodag/plugins/authentication/sas_auth.py b/eodag/plugins/authentication/sas_auth.py index e6040d238..672ead262 100644 --- a/eodag/plugins/authentication/sas_auth.py +++ b/eodag/plugins/authentication/sas_auth.py @@ -86,16 +86,19 @@ def __call__(self, request: PreparedRequest) -> PreparedRequest: class SASAuth(Authentication): """SASAuth authentication plugin + An apiKey that is added in the headers can be given in the credentials in the config file. - The configuration parameters for this plugin are: + :param provider: provider name + :param config: Authentication plugin configuration: - * **type** [str] (mandatory): SASAuth - * **auth_uri** [str] (mandatory): url used to get the signed url - * **signed_url_key** [str] (mandatory): key to get the signed url - * **headers** [Dict[str, str]] (mandatory if apiKey is used): headers to be added to the requests - * **ssl_verify** [bool]: if the ssl certificates should be verified in the requests; default: True + * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): SASAuth + * :attr:`~eodag.config.PluginConfig.auth_uri` (``str``) (**mandatory**): url used to get the signed url + * :attr:`~eodag.config.PluginConfig.signed_url_key (``str``) (**mandatory**): key to get the signed url + * :attr:`~eodag.config.PluginConfig.headers** (``Dict[str, str]``): (**mandatory if apiKey is used**) + headers to be added to the requests + * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be + verified in the requests; default: True - An apiKey that is added in the headers can be given in the credentials in the config file. """ def validate_config_credentials(self) -> None: diff --git a/eodag/plugins/authentication/token.py b/eodag/plugins/authentication/token.py index 55cc8e575..72f14fc02 100644 --- a/eodag/plugins/authentication/token.py +++ b/eodag/plugins/authentication/token.py @@ -43,19 +43,26 @@ class TokenAuth(Authentication): """TokenAuth authentication plugin - fetches a token which is added to search/download requests - The configuration parameters for this plugin are: - - * **type** [str] (mandatory): TokenAuth - * **auth_uri** [str] (mandatory): url used to fetch the access token with user/password - * **refresh_uri** [str] : url used to fetch the access token with a refresh token - * **token_type** [str]: type of the token (json or text); default: text - * **token_key** [str] (mandatory if token_type=json): key to get the access token in the - response to the token request - * **refresh_token_key** [str]: key to get the refresh token in the response to the token request - * **ssl_verify** [bool]: if the ssl certificates should be verified in the requests; default: True - * **auth_error_code** [int]: which error code is returned in case of an authentication error - * **req_data** [Dict[str, Any]]: if the credentials should be sent as data in the post request, - the json structure can be given in this parameter + :param provider: provider name + :param config: Authentication plugin configuration: + + * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): TokenAuth + * :attr:`~eodag.config.PluginConfig.auth_uri` (``str``) (**mandatory**): url used to fetch + the access token with user/password + * :attr:`~eodag.config.PluginConfig.refresh_uri` (``str``) : url used to fetch the + access token with a refresh token + * :attr:`~eodag.config.PluginConfig.token_type` (``str``): type of the token (``json`` + or ``text``); default: ``text`` + * :attr:`~eodag.config.PluginConfig.token_key` (``str``): (mandatory if token_type=json) + key to get the access token in the response to the token request + * :attr:`~eodag.config.PluginConfig.refresh_token_key` (``str``): key to get the refresh + token in the response to the token request + * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates + should be verified in the requests; default: ``True`` + * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is + returned in case of an authentication error + * :attr:`~eodag.config.PluginConfig.req_data` (``Dict[str, Any]``): if the credentials + should be sent as data in the post request, the json structure can be given in this parameter """ def __init__(self, provider: str, config: PluginConfig) -> None: diff --git a/eodag/plugins/authentication/token_exchange.py b/eodag/plugins/authentication/token_exchange.py index 52dc24b68..2a3c12081 100644 --- a/eodag/plugins/authentication/token_exchange.py +++ b/eodag/plugins/authentication/token_exchange.py @@ -38,16 +38,22 @@ class OIDCTokenExchangeAuth(Authentication): """Token exchange implementation using :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` token as subject. - The configuration keys of this plugin are as follows (they have no defaults): + :param provider: provider name + :param config: Authentication plugin configuration: - * **subject** [Dict[str, Any]] (mandatory): The full OIDCAuthorizationCodeFlowAuth plugin - configuration used to retrieve subject token - * **subject_issuer** [str] (mandatory): Identifies the issuer of the subject_token - * **token_uri** [str] (mandatory): The url to query to get the authorized token - * **client_id** [str] (mandatory): The OIDC provider's client ID of the eodag provider - * **audience** [str] (mandatory): This parameter specifies the target client you want the new token minted for. - * **token_key** [str] (mandatory): The key pointing to the token in the json response to - the POST request to the token server + * :attr:`~eodag.config.PluginConfig.subject` (``Dict[str, Any]``) (**mandatory**): + The full :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` plugin + configuration used to retrieve subject token + * :attr:`~eodag.config.PluginConfig.subject_issuer` (``str``) (**mandatory**): Identifies + the issuer of the subject_token + * :attr:`~eodag.config.PluginConfig.token_uri` (``str``) (**mandatory**): The url to + query to get the authorized token + * :attr:`~eodag.config.PluginConfig.client_id` (``str``) (**mandatory**): The OIDC + provider's client ID of the eodag provider + * :attr:`~eodag.config.PluginConfig.audience` (``str``) (**mandatory**): This parameter + specifies the target client you want the new token minted for. + * :attr:`~eodag.config.PluginConfig.token_key` (``str``) (**mandatory**): The key + pointing to the token in the json response to the POST request to the token server """ diff --git a/eodag/plugins/crunch/filter_date.py b/eodag/plugins/crunch/filter_date.py index e52535693..dc67bca9b 100644 --- a/eodag/plugins/crunch/filter_date.py +++ b/eodag/plugins/crunch/filter_date.py @@ -39,8 +39,10 @@ class FilterDate(Crunch): The Crunch configuration, may contain : - * **start** [str]: start sensing time in iso format - * **end** [str]: end sensing time in iso format + :param config: Search plugin configuration: + + * :attr:`~eodag.config.PluginConfig.start` (``str``): start sensing time in iso format + * :attr:`~eodag.config.PluginConfig.end` (``str``): end sensing time in iso format """ @staticmethod diff --git a/eodag/plugins/crunch/filter_latest_tpl_name.py b/eodag/plugins/crunch/filter_latest_tpl_name.py index 53dd9c1d2..a98f0821d 100644 --- a/eodag/plugins/crunch/filter_latest_tpl_name.py +++ b/eodag/plugins/crunch/filter_latest_tpl_name.py @@ -37,7 +37,9 @@ class FilterLatestByName(Crunch): The Crunch configuration must contain : - * **name_pattern** [str] (mandatory) : product name pattern + :param config: Search plugin configuration: + + * :attr:`~eodag.config.PluginConfig.name_pattern` (``str``) (**mandatory**): product name pattern """ NAME_PATTERN_CONSTRAINT = re.compile(r"\(\?P\\d\{6\}\)") diff --git a/eodag/plugins/crunch/filter_overlap.py b/eodag/plugins/crunch/filter_overlap.py index b5a7d25f6..c321952c4 100644 --- a/eodag/plugins/crunch/filter_overlap.py +++ b/eodag/plugins/crunch/filter_overlap.py @@ -40,14 +40,19 @@ class FilterOverlap(Crunch): Filter products, retaining only those that are overlapping with the search_extent - The Crunch configuration may contain : + The Crunch configuration may contain the following parameters which are mutually exclusive: - * **minimum_overlap** [Union[float, str]]: minimal overlap percentage; default: "0" - * **contains** [bool]: True if product geometry contains the search area; default: False - * **intersects** [bool]: True if product geometry intersects the search area; default: False - * **within** [bool]: True if product geometry is within the search area; default: False + :param config: Search plugin configuration: + + * :attr:`~eodag.config.PluginConfig.minimum_overlap` (``Union[float, str]``): minimal overlap + percentage; default: ``"0"`` + * :attr:`~eodag.config.PluginConfig.contains` (``bool``): True if product geometry + contains the search area; default: ``False`` + * :attr:`~eodag.config.PluginConfig.intersects` (``bool``): True if product geometry + intersects the search area; default: ``False`` + * :attr:`~eodag.config.PluginConfig.within` (``bool``): True if product geometry is + within the search area; default: ``False`` - These configuration parameters are mutually exclusive. """ def proceed( diff --git a/eodag/plugins/crunch/filter_property.py b/eodag/plugins/crunch/filter_property.py index 4e70ffd93..ea958fcb4 100644 --- a/eodag/plugins/crunch/filter_property.py +++ b/eodag/plugins/crunch/filter_property.py @@ -34,10 +34,12 @@ class FilterProperty(Crunch): Filter products, retaining only those whose property match criteria - The Crunch configuration should contain : + :param config: Search plugin configuration: - * **** [Any] (mandatory): property key from product.properties, associated to its filter value - * **operator** [str]: Operator used for filtering (one of `lt,le,eq,ne,ge,gt`). Default is `eq` + * ```` ``(Any)`` (**mandatory**): property key from product.properties, + associated to its filter value + * :attr:`~eodag.config.PluginConfig.operator` (``str``): Operator used for filtering + (one of ``lt,le,eq,ne,ge,gt``). Default is ``eq`` """ def proceed( diff --git a/eodag/plugins/download/aws.py b/eodag/plugins/download/aws.py index aeb9c6dd7..ee13568c0 100644 --- a/eodag/plugins/download/aws.py +++ b/eodag/plugins/download/aws.py @@ -214,24 +214,30 @@ class AwsDownload(Download): """Download on AWS using S3 protocol. - The configuration parameters for this plugin are: - - * **type** [str] (mandatory): AwsDownload - * **base_uri** [str] (mandatory): s3 endpoint url - * **requester_pays** [bool]: whether download is done from a requester-pays bucket or not; default: False - * **flatten_top_dirs** [bool]: if the directory structure should be flattened; default: True - * **ignore_assets** [bool]: ignore assets and download using downloadLink; default: False - * **ssl_verify** [bool]: if the ssl certificates should be verified in requests; default: True - * **bucket_path_level** [int]: at which level of the path part of the url the bucket can be found; - If no bucket_path_level is given, the bucket is taken from the first element of the netloc part. - * **products** [Dict[str, Dict[str, Any]]: product type specific config; the keys are the product types, - the values are dictionaries which can contain the keys: - - * **default_bucket** [str]: bucket where the product type can be found - * **complementary_url_key** [str]: keys to add additional urls - * **build_safe** [bool]: if a SAFE (Standard Archive Format for Europe) product should - be created; used for Sentinel products; default: False - * **fetch_metadata** [Dict[str, Any]: config for metadata to be fetched for the SAFE product + :param provider: provider name + :param config: Download plugin configuration: + + * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): AwsDownload + * :attr:`~eodag.config.PluginConfig.base_uri` (``str``) (**mandatory**): s3 endpoint url + * :attr:`~eodag.config.PluginConfig.requester_pays` (``bool``): whether download is done + from a requester-pays bucket or not; default: ``False`` + * :attr:`~eodag.config.PluginConfig.flatten_top_dirs` (``bool``): if the directory structure + should be flattened; default: ``True`` + * :attr:`~eodag.config.PluginConfig.ignore_assets` (``bool``): ignore assets and download + using downloadLink; default: ``False`` + * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should + be verified in requests; default: ``True`` + * :attr:`~eodag.config.PluginConfig.bucket_path_level` (``int``): at which level of the + path part of the url the bucket can be found; If no bucket_path_level is given, the bucket + is taken from the first element of the netloc part. + * :attr:`~eodag.config.PluginConfig.products` (``Dict[str, Dict[str, Any]``): product type + specific config; the keys are the product types, the values are dictionaries which can contain the keys: + + * **default_bucket** (``str``): bucket where the product type can be found + * **complementary_url_key** (``str``): keys to add additional urls + * **build_safe** (``bool``): if a SAFE (Standard Archive Format for Europe) product should + be created; used for Sentinel products; default: False + * **fetch_metadata** (``Dict[str, Any]``): config for metadata to be fetched for the SAFE product """ diff --git a/eodag/plugins/download/creodias_s3.py b/eodag/plugins/download/creodias_s3.py index b4ad5cb61..5ee504543 100644 --- a/eodag/plugins/download/creodias_s3.py +++ b/eodag/plugins/download/creodias_s3.py @@ -26,12 +26,15 @@ class CreodiasS3Download(AwsDownload): """ Download on creodias s3 from their VMs (extension of AwsDownload) - The configuration parameters for this plugin are: - * **type** [str] (mandatory): CreodiasS3Download - * **base_uri** [str] (mandatory): s3 endpoint url - * **s3_bucket** [str] (mandatory): bucket where the products can be found - * **ssl_verify** [bool]: if the ssl certificates should be verified in requests; default: True + :param provider: provider name + :param config: Download plugin configuration: + + * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): CreodiasS3Download + * :attr:`~eodag.config.PluginConfig.base_uri` (``str``) (**mandatory**): s3 endpoint url + * :attr:`~eodag.config.PluginConfig.s3_bucket` (``str``) (**mandatory**): bucket where the products can be found + * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be + verified in requests; default: ``True`` """ def _get_authenticated_objects_unsigned(self, bucket_name, prefix, auth_dict): diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py index 36adf1a6b..3337db495 100644 --- a/eodag/plugins/download/http.py +++ b/eodag/plugins/download/http.py @@ -101,7 +101,7 @@ class HTTPDownload(Download): :param provider: provider name :param config: Download plugin configuration: - * :attr:`~eodag.config.PluginConfig.type` (``str``): (mandatory) ``HTTPDownload`` + * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): ``HTTPDownload`` * :attr:`~eodag.config.PluginConfig.base_uri` (``str``): default endpoint url * :attr:`~eodag.config.PluginConfig.method` (``str``): HTTP request method for the download request (``GET`` or ``POST``); default: ``GET`` @@ -109,8 +109,8 @@ class HTTPDownload(Download): extracted; default: ``True`` * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an authentication error - * :attr:`~eodag.config.PluginConfig.dl_url_params` [Dict[str, Any]]: parameters to be added to the query params - of the request + * :attr:`~eodag.config.PluginConfig.dl_url_params` (``Dict[str, Any]``): parameters to be + added to the query params of the request * :attr:`~eodag.config.PluginConfig.archive_depth` (``int``): level in extracted path tree where to find data; default: ``1`` * :attr:`~eodag.config.PluginConfig.flatten_top_dirs` (``bool``): if the directory structure should be @@ -126,8 +126,8 @@ class HTTPDownload(Download): * :attr:`~eodag.config.PluginConfig.no_auth_download` (``bool``): if the download should be done without authentication; default: ``True`` * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to be ordered to download it; - if this paramter is set to true, a mapping for the orderLink has to be added to the metadata mapping of - the search plugin used for the provider; default: False + if this parameter is set to true, a mapping for the orderLink has to be added to the metadata mapping of + the search plugin used for the provider; default: ``False`` * :attr:`~eodag.config.PluginConfig.order_method` (``str``): HTTP request method for the order request (``GET`` or ``POST``); default: ``GET`` * :attr:`~eodag.config.PluginConfig.order_headers` (``[Dict[str, str]]``): headers to be added to the order diff --git a/eodag/plugins/download/s3rest.py b/eodag/plugins/download/s3rest.py index 43de0e4e0..6141d6742 100644 --- a/eodag/plugins/download/s3rest.py +++ b/eodag/plugins/download/s3rest.py @@ -67,19 +67,22 @@ class S3RestDownload(Download): Re-use AwsDownload bucket some handling methods - Download plugin configuration: - - * **config.base_uri** [str] (mandatory): default endpoint url - * **config.extract** [bool]: extract downloaded archive or not - * **config.auth_error_code** [int]: authentication error code - * **config.bucket_path_level** [int]: bucket location index in path.split('/') - * **config.order_enabled** [bool]: whether order is enabled or not if product is `OFFLINE` - * **config.order_method** [str]: HTTP request method, GET (default) or POST - * **config.order_headers** [dict]: order request headers - * **order_on_response** [:class:`~eodag.config.PluginConfig.OrderOnResponse`]: a typed dictionary - containing the key 'metadata_mapping' which can be used to add new product properties - based on the data in response to the order request - * **config.order_status** [:class:`~eodag.config.PluginConfig.OrderStatus`]: Order status handling + :param provider: provider name + :param config: Download plugin configuration: + + * :attr:`~eodag.config.PluginConfig.base_uri` (``str``) (**mandatory**): default endpoint url + * :attr:`~eodag.config.PluginConfig.extract` (``bool``): extract downloaded archive or not + * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): authentication error code + * :attr:`~eodag.config.PluginConfig.bucket_path_level` (``int``): bucket location index in path.split('/') + * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): whether order is enabled + or not if product is `OFFLINE` + * :attr:`~eodag.config.PluginConfig.order_method` (``str``) HTTP request method, ``GET`` (default) or ``POST`` + * :attr:`~eodag.config.PluginConfig.order_headers` (``[Dict[str, str]]``): order request headers + * :attr:`~eodag.config.PluginConfig.order_on_response` (:class:`~eodag.config.PluginConfig.OrderOnResponse`): + a typed dictionary containing the key 'metadata_mapping' which can be used to add new product properties + based on the data in response to the order request + * :attr:`~eodag.config.PluginConfig.order_status` (:class:`~eodag.config.PluginConfig.OrderStatus`): + Order status handling """ def __init__(self, provider: str, config: PluginConfig) -> None: diff --git a/eodag/plugins/search/build_search_result.py b/eodag/plugins/search/build_search_result.py index b44c6f449..accfc81db 100644 --- a/eodag/plugins/search/build_search_result.py +++ b/eodag/plugins/search/build_search_result.py @@ -81,11 +81,14 @@ class BuildPostSearchResult(PostJsonSearch): performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult` object. - The available configuration parameters inherits from parent classes (PostJsonSearch and + The available configuration parameters are inherited from parent classes (PostJsonSearch and QueryStringSearch), with particularly for this plugin: - * **remove_from_query** [List[str]]: List of parameters used to parse metadata but that must - not be included to the query + :param provider: provider name + :param config: Search plugin configuration: + + * :attr:`~eodag.config.PluginConfig.remove_from_query` (``List[str]``): List of parameters + used to parse metadata but that must not be included to the query """ @@ -243,9 +246,12 @@ class BuildSearchResult(BuildPostSearchResult): The available configuration parameters inherits from parent classes (BuildPostSearchResult, PostJsonSearch and QueryStringSearch), with particularly for this plugin: - * **end_date_excluded** [bool]: Set to `False` if provider does not include end date in - the search request; In this case, if the end date is at midnight, the previous day will be - used. default: true + :param provider: provider name + :param config: Search plugin configuration: + + * :attr:`~eodag.config.PluginConfig.end_date_excluded` (``bool``): Set to ``False`` if provider + does not include end date in the search request; In this case, if the end date is at midnight, + the previous day will be used. default: ``True`` """ diff --git a/eodag/plugins/search/cop_marine.py b/eodag/plugins/search/cop_marine.py index a37c9493a..28124c2bc 100644 --- a/eodag/plugins/search/cop_marine.py +++ b/eodag/plugins/search/cop_marine.py @@ -114,8 +114,10 @@ class CopMarineSearch(StaticStacSearch): a special method which fetches the urls of the available products from an S3 storage and filters them has been written. The configuration parameters are inherited from the parent and grand-parent classes. The - `auto_discovery` parameter in the `discover_metadata` section has to be set to `false` and the - `fetch_url` in the `discover_queryables` queryables section has to be set to `null` to + :attr:`~eodag.config.PluginConfig.DiscoverMetadata.auto_discovery` parameter in the + :attr:`~eodag.config.PluginConfig.discover_metadata` section has to be set to ``false`` and the + :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` in the + :attr:`~eodag.config.PluginConfig.discover_queryables` queryables section has to be set to ``null`` to overwrite the default config from the stac provider configuration because those functionalities are not available. """ diff --git a/eodag/plugins/search/creodias_s3.py b/eodag/plugins/search/creodias_s3.py index d47fd6a42..3c34f61d0 100644 --- a/eodag/plugins/search/creodias_s3.py +++ b/eodag/plugins/search/creodias_s3.py @@ -121,7 +121,10 @@ class CreodiasS3Search(ODataV4Search): adapts results so that the assets contain links to s3. It has the same configuration parameters as ODataV4Search and one additional parameter: - * **s3_endpoint** [str] (mandatory): base url of the s3 + :param provider: provider name + :param config: Search plugin configuration: + + * :attr:`~eodag.config.PluginConfig.s3_endpoint` (``str``) (**mandatory**): base url of the s3 """ def __init__(self, provider, config): diff --git a/eodag/plugins/search/csw.py b/eodag/plugins/search/csw.py index 214da4f72..c67cedbef 100644 --- a/eodag/plugins/search/csw.py +++ b/eodag/plugins/search/csw.py @@ -53,41 +53,45 @@ class CSWSearch(Search): """A plugin for implementing search based on OGC CSW - It has the following configuration parameters: - * **api_endpoint** [str] (mandatory): The endpoint of the provider's search interface - * **version** [str]: OGC Catalogue Service version; default: 2.0.2 - * **search_definition** [Dict[str, Any]] (mandatory): + :param provider: provider name + :param config: Search plugin configuration: - * **product_type_tags** [List[Dict[str, Any]]: dict of product type tags - * **resource_location_filter** [str]: regex string - * **date_tags** [Dict[str, Any]]: tags for start and end + * :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): The endpoint of the + provider's search interface + * :attr:`~eodag.config.PluginConfig.version` (``str``): OGC Catalogue Service version; default: ``2.0.2`` + * :attr:`~eodag.config.PluginConfig.search_definition` (``Dict[str, Any]``) (**mandatory**): - * **metadata_mapping** [Dict[str, Any]]: The search plugins of this kind can detect when a - metadata mapping is "query-able", and get the semantics of how to format the query string - parameter that enables to make a query on the corresponding metadata. To make a metadata query-able, - just configure it in the metadata mapping to be a list of 2 items, the first one being the - specification of the query string search formatting. The later is a string following the - specification of Python string formatting, with a special behaviour added to it. For example, - an entry in the metadata mapping of this kind:: + * **product_type_tags** (``List[Dict[str, Any]``): dict of product type tags + * **resource_location_filter** (``str``): regex string + * **date_tags** (``Dict[str, Any]``): tags for start and end - completionTimeFromAscendingNode: - - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}' - - '$.properties.acquisition.endViewingDate' + * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Any]``): The search plugins of this kind can + detect when a metadata mapping is "query-able", and get the semantics of how to format the query string + parameter that enables to make a query on the corresponding metadata. To make a metadata query-able, + just configure it in the metadata mapping to be a list of 2 items, the first one being the + specification of the query string search formatting. The later is a string following the + specification of Python string formatting, with a special behaviour added to it. For example, + an entry in the metadata mapping of this kind:: + + completionTimeFromAscendingNode: + - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}' + - '$.properties.acquisition.endViewingDate' + + means that the search url will have a query string parameter named *"f"* with a value of + *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done with the value + of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that + ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value + of ``completionTimeFromAscendingNode``. This example shows all there is to know about the + semantics of the query string formatting introduced by this plugin: any eodag search parameter + can be referenced in the query string with an additional optional conversion function that + is separated from it by a ``#`` (see :func:`~eodag.api.product.metadata_mapping.format_metadata` for further + details on the available converters). Note that for the values in the + :attr:`~eodag.config.PluginConfig.free_text_search_operations` configuration parameter follow the same rule. + If the metadata_mapping is not a list but only a string, this means that the parameters is not queryable but + it is included in the result obtained from the provider. The string indicates how the provider result should + be mapped to the eodag parameter. - means that the search url will have a query string parameter named *"f"* with a value of - *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done with the value - of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that - ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value - of ``completionTimeFromAscendingNode``. This example shows all there is to know about the - semantics of the query string formatting introduced by this plugin: any eodag search parameter - can be referenced in the query string with an additional optional conversion function that - is separated from it by a ``#`` (see :func:`~eodag.utils.format_metadata` for further details - on the available converters). Note that for the values in the ``free_text_search_operations`` - configuration parameter follow the same rule. If the metadata_mapping is not a list but - only a string, this means that the parameters is not queryable but it is included in the - result obtained from the provider. The string indicates how the provider result should be - mapped to the eodag parameter. """ def __init__(self, provider: str, config: PluginConfig) -> None: diff --git a/eodag/plugins/search/data_request_search.py b/eodag/plugins/search/data_request_search.py index 1ea9d9e4b..429aab341 100644 --- a/eodag/plugins/search/data_request_search.py +++ b/eodag/plugins/search/data_request_search.py @@ -63,79 +63,101 @@ class DataRequestSearch(Search): #. check the status of the request job #. if finished - fetch the result of the job - It has the following configuration parameters: - - * **api_endpoint** [str] (mandatory): The endpoint of the provider's search interface - * **results_entry** [str] (mandatory): The name of the key in the provider search - result that gives access to the result entries - * **data_request_url** [str] (mandatory): url to which the data request shall be sent - * **status_url** [str] (mandatory): url to fetch the status of the data request - * **result_url** [str] (mandatory): url to fetch the search result when the data request is done - * **need_auth** [bool]: if authentication is needed for the search request; default: False - * **auth_error_code** [int]: which error code is returned in case of an authentication - error; only used if `need_auth=true` - * **ssl_verify** [bool]: if the ssl certificates should be verified in requests; default: True - * **timeout** [int]: time to wait until request timeout in seconds; default: 5 - * **dates_required** [bool]: if date parameters are mandatory in the request; default: True - * **pagination** [:class:`~eodag.config.PluginConfig.Pagination`] (mandatory): The configuration of - how the pagination is done on the provider. It is a tree with the following nodes: - - * **total_items_nb_key_path** [str]: An XPath or JsonPath leading to the total number of - results satisfying a request. This is used for providers which provides the total results - metadata along with the result of the query and don't have an endpoint for querying - the number of items satisfying a request, or for providers for which the count endpoint - returns a json or xml document - * **max_items_per_page** [int]: The maximum number of items per page that the provider can - handle; default: 50 - * **start_page** [int]: number of the first page; default: 1 - * **discover_product_types** [Dict[str, Any]]: configuration for product type discovery - based on information from the provider; It contains the keys: - - * **fetch_url** [str] (mandatory): url from which the product types can be fetched - * **result_type** [str]: type of the provider result; currently only "json" is supported, - (other types could be used in an extension of this plugin) - * **results_entry** [str] (mandatory): json path to the list of product types - * **generic_product_type_id** [str]: mapping for the product type id - * **generic_product_type_parsable_metadata** [Dict[str, str]]: mapping for product type - metadata (e.g. abstract, licence) which can be parsed from the provider result - * **generic_product_type_parsable_properties** [Dict[str, str]]: mapping for product type - properties which can be parsed from the result that are not product type metadata - * **single_collection_fetch_url** [str]: url to fetch data for a single collection; - used if product type metadata is not available from the endpoint given in `fetch_url` - * **single_collection_fetch_qs** [str]: query string to be added to the `fetch_url` - to filter for a collection - * **single_product_type_parsable_metadata** [Dict[str, str]]: mapping for product type - metadata returned by the endpoint given in `single_collection_fetch_url`. - * **constraints_file_url** [str]: url to fetch the constraints for a specific product type, - can be an http url or a path to a file; the constraints are used to build queryables - * **constraints_entry** [str]: key in the json result where the constraints can be found; - if not given, it is assumed that the constraints are on top level of the result, i.e. - the result is an array of constraints - * **metadata_mapping** [Dict[str, Any]]: The search plugins of this kind can detect when a - metadata mapping is "query-able", and get the semantics of how to format the query string - parameter that enables to make a query on the corresponding metadata. To make a metadata query-able, - just configure it in the metadata mapping to be a list of 2 items, the first one being the - specification of the query string search formatting. The later is a string following the - specification of Python string formatting, with a special behaviour added to it. For example, - an entry in the metadata mapping of this kind:: - - completionTimeFromAscendingNode: - - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}' - - '$.properties.acquisition.endViewingDate' - - means that the search url will have a query string parameter named *"f"* with a value of - *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done with the value - of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that - ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value - of ``completionTimeFromAscendingNode``. This example shows all there is to know about the - semantics of the query string formatting introduced by this plugin: any eodag search parameter - can be referenced in the query string with an additional optional conversion function that - is separated from it by a ``#`` (see :func:`~eodag.utils.format_metadata` for further details - on the available converters). Note that for the values in the ``free_text_search_operations`` - configuration parameter follow the same rule. If the metadata_mapping is not a list but - only a string, this means that the parameters is not queryable but it is included in the - result obtained from the provider. The string indicates how the provider result should be - mapped to the eodag parameter. + :param provider: provider name + :param config: Search plugin configuration: + + * :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): The endpoint of the + provider's search interface + * :attr:`~eodag.config.PluginConfig.results_entry` (``str``) (**mandatory**): The name of + the key in the provider search result that gives access to the result entries + * :attr:`~eodag.config.PluginConfig.data_request_url` (``str``) (**mandatory**): url + to which the data request shall be sent + * :attr:`~eodag.config.PluginConfig.status_url` (``str``) (**mandatory**): url to fetch + the status of the data request + * :attr:`~eodag.config.PluginConfig.result_url` (``str``) (**mandatory**): url to fetch + the search result when the data request is done + * :attr:`~eodag.config.PluginConfig.need_auth` (``bool``): if authentication is needed for + the search request; default: ``False`` + * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an + authentication error; only used if ``need_auth=true`` + * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be + verified in requests; default: ``True`` + * :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds; + default: ``5`` + * :attr:`~eodag.config.PluginConfig.dates_required` (``bool``): if date parameters are mandatory + in the request; default: ``True`` + * :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`) + (**mandatory**): The configuration of how the pagination is done on the provider. It is a tree with the + following nodes: + + * :attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path` (``str``): An XPath or JsonPath + leading to the total number of results satisfying a request. This is used for providers which provides the + total results metadata along with the result of the query and don't have an endpoint for querying + the number of items satisfying a request, or for providers for which the count endpoint + returns a json or xml document + * :attr:`~eodag.config.PluginConfig.Pagination.max_items_per_page` (``int``): The maximum + number of items per page that the provider can handle; default: ``50`` + * :attr:`~eodag.config.PluginConfig.Pagination.start_page` (``int``): number of the + first page; default: ``1`` + + * :attr:`~eodag.config.PluginConfig.discover_product_types` + (:class:`~eodag.config.PluginConfig.DiscoverProductTypes`): configuration for product type discovery based on + information from the provider; It contains the keys: + + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` (``str``) (**mandatory**): url from which + the product types can be fetched + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.result_type` (``str``): type of the provider result; + currently only ``json`` is supported (other types could be used in an extension of this plugin) + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.results_entry` (``str``) (**mandatory**): json path + to the list of product types + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_id` (``str``): mapping for the + product type id + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_metadata` + (``Dict[str, str]``): mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed + from the provider result + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_properties` + (``Dict[str, str]``): mapping for product type properties which can be parsed from the result that are not + product type metadata + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url` (``str``): url to fetch + data for a single collection; used if product type metadata is not available from the endpoint given in + :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_qs` (``str``): query string + to be added to the :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` to filter for a + collection + * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_product_type_parsable_metadata` + (``Dict[str, str]``): mapping for product type metadata returned by the endpoint given in + :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url`. + + * :attr:`~eodag.config.PluginConfig.constraints_file_url` (``str``): url to fetch the constraints for a specific + product type, can be an http url or a path to a file; the constraints are used to build queryables + * :attr:`~eodag.config.PluginConfig.constraints_entry` (``str``): key in the json result where the constraints + can be found; if not given, it is assumed that the constraints are on top level of the result, i.e. + the result is an array of constraints + * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Any]``): The search plugins of this kind can + detect when a metadata mapping is "query-able", and get the semantics of how to format the query string + parameter that enables to make a query on the corresponding metadata. To make a metadata query-able, + just configure it in the metadata mapping to be a list of 2 items, the first one being the + specification of the query string search formatting. The later is a string following the + specification of Python string formatting, with a special behaviour added to it. For example, + an entry in the metadata mapping of this kind:: + + completionTimeFromAscendingNode: + - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}' + - '$.properties.acquisition.endViewingDate' + + means that the search url will have a query string parameter named *"f"* with a value of + *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done with the value + of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that + ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value + of ``completionTimeFromAscendingNode``. This example shows all there is to know about the + semantics of the query string formatting introduced by this plugin: any eodag search parameter + can be referenced in the query string with an additional optional conversion function that + is separated from it by a ``#`` (see :func:`~eodag.api.product.metadata_mapping.format_metadata` for further + details on the available converters). Note that for the values in the + :attr:`~eodag.config.PluginConfig.free_text_search_operations` configuration parameter follow the same rule. + If the metadata_mapping is not a list but only a string, this means that the parameters is not queryable but + it is included in the result obtained from the provider. The string indicates how the provider result should + be mapped to the eodag parameter. """ data_request_id: Optional[str] diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index bbbe47880..2fa0bc03f 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -246,7 +246,7 @@ class QueryStringSearch(Search): * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.metadata_pattern` (``str``): regex string a parameter in the result should match so that is used * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.search_param` (``Union [str, Dict[str, Any]]``): format - to add a query param given by the user and not in the metadata mapping to the requets, 'metadata' will be + to add a query param given by the user and not in the metadata mapping to the requests, 'metadata' will be replaced by the search param; can be a string or a dict containing :attr:`~eodag.config.PluginConfig.free_text_search_operations` (see :class:`~eodag.plugins.search.qssearch.ODataV4Search`) @@ -1395,17 +1395,20 @@ class PostJsonSearch(QueryStringSearch): not a get request, the pagination configuration will look slightly different. It has the following parameters: - * **next_page_query_obj**: The additional parameters needed to add pagination information to - the search request. These parameters won't be included in result. This must be a json dict - formatted like `{{"foo":"bar"}}` because it will be passed to a `.format()` method - before being loaded as json. - * **total_items_nb_key_path** [str]: An XPath or JsonPath leading to the total number of - results satisfying a request. This is used for providers which provides the total results - metadata along with the result of the query and don't have an endpoint for querying - the number of items satisfying a request, or for providers for which the count endpoint - returns a json or xml document - * **max_items_per_page** [int]: The maximum number of items per page that the provider can - handle; default: 50 + :param provider: provider name + :param config: Search plugin configuration: + + * :attr:`~eodag.config.PluginConfig.next_page_query_obj` (``str``): The additional parameters + needed to add pagination information to the search request. These parameters won't be + included in result. This must be a json dict formatted like `{{"foo":"bar"}}` because + it will be passed to a `.format()` method before being loaded as json. + * :attr:`~eodag.config.PluginConfig.total_items_nb_key_path*` (``str``): An XPath or JsonPath + leading to the total number of results satisfying a request. This is used for providers + which provides the total results metadata along with the result of the query and don't + have an endpoint for querying the number of items satisfying a request, or for providers + for which the count endpoint returns a json or xml document + * :attr:`~eodag.config.PluginConfig.max_items_per_page` (``int``): The maximum number of items + per page that the provider can handle; default: ``50`` """ @@ -1797,8 +1800,9 @@ class StacSearch(PostJsonSearch): (see stac_provider.yml). If some parameters are different for a specific provider, they have to be overwritten. If certain functionalities are not available, their configuration parameters have to be overwritten with `null`. E.g. if there is no queryables endpoint, - the `fetch_url` and `product_type_fetch_url` in the `discover_queryables` config have - to be set to `null`. + the :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` and + :attr:`~eodag.config.PluginConfig.DiscoverQueryables.product_type_fetch_url` in the + :attr:`~eodag.config.PluginConfig.discover_queryables` config have to be set to ``null``. """ def __init__(self, provider: str, config: PluginConfig) -> None: diff --git a/eodag/plugins/search/static_stac_search.py b/eodag/plugins/search/static_stac_search.py index f7a500ec5..49e52771e 100644 --- a/eodag/plugins/search/static_stac_search.py +++ b/eodag/plugins/search/static_stac_search.py @@ -50,10 +50,15 @@ class StaticStacSearch(StacSearch): The plugin inherits the configuration parameters from PostJsonSearch (via the StacSearch inheritance) with the following particularities: - * **api_endpoint** [str] (mandatory): path to the catalog; in contrast to the api_endpoint for - other plugin types this can be a url or local system path. - * **max_connections** [int]: Maximum number of connections for HTTP requests; default: 100. - * **timeout** [int]: Timeout in seconds for each internal HTTP request; default: 5 + :param provider: provider name + :param config: Search plugin configuration: + + * :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): path to the catalog; + in contrast to the api_endpoint for other plugin types this can be a url or local system path. + * :attr:`~eodag.config.PluginConfig.max_connections` (``int``): Maximum number of + connections for HTTP requests; default: ``100`` + * :attr:`~eodag.config.PluginConfig.timeout` (``int``): Timeout in seconds for each + internal HTTP request; default: ``5`` """ From ac73a14cf29b0a310f3efea40c6c0657a5776ed3 Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Tue, 1 Oct 2024 18:13:25 +0200 Subject: [PATCH 16/25] docs: fix docstring formatting issues --- eodag/plugins/authentication/openid_connect.py | 1 + eodag/plugins/search/data_request_search.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/eodag/plugins/authentication/openid_connect.py b/eodag/plugins/authentication/openid_connect.py index b133dbc14..84083ffc0 100644 --- a/eodag/plugins/authentication/openid_connect.py +++ b/eodag/plugins/authentication/openid_connect.py @@ -238,6 +238,7 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase): Refers to the name of the query param to be used in the query request * :attr:`~eodag.config.PluginConfig.refresh_token_key` (``str``): The key pointing to the refresh_token in the json response to the POST request to the token server + """ SCOPE = "openid" diff --git a/eodag/plugins/search/data_request_search.py b/eodag/plugins/search/data_request_search.py index 429aab341..d1d67b836 100644 --- a/eodag/plugins/search/data_request_search.py +++ b/eodag/plugins/search/data_request_search.py @@ -71,7 +71,7 @@ class DataRequestSearch(Search): * :attr:`~eodag.config.PluginConfig.results_entry` (``str``) (**mandatory**): The name of the key in the provider search result that gives access to the result entries * :attr:`~eodag.config.PluginConfig.data_request_url` (``str``) (**mandatory**): url - to which the data request shall be sent + to which the data request shall be sent * :attr:`~eodag.config.PluginConfig.status_url` (``str``) (**mandatory**): url to fetch the status of the data request * :attr:`~eodag.config.PluginConfig.result_url` (``str``) (**mandatory**): url to fetch @@ -158,6 +158,7 @@ class DataRequestSearch(Search): If the metadata_mapping is not a list but only a string, this means that the parameters is not queryable but it is included in the result obtained from the provider. The string indicates how the provider result should be mapped to the eodag parameter. + """ data_request_id: Optional[str] From fc48ed008708217a535bf5e4555653a70e41f3e5 Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Wed, 2 Oct 2024 09:06:18 +0200 Subject: [PATCH 17/25] docs: fix docstring format error --- eodag/plugins/authentication/openid_connect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eodag/plugins/authentication/openid_connect.py b/eodag/plugins/authentication/openid_connect.py index 84083ffc0..32ceb3e55 100644 --- a/eodag/plugins/authentication/openid_connect.py +++ b/eodag/plugins/authentication/openid_connect.py @@ -195,7 +195,7 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase): to exchange the authorization code obtained from the OIDC provider for an authorized token * :attr:`~eodag.config.PluginConfig.client_id` (``str``) (**mandatory**): The OIDC provider's client ID of the eodag provider - * :attr:`~eodag.config.PluginConfig.user_consent_needed` (``bool`) (mandatory): Whether + * :attr:`~eodag.config.PluginConfig.user_consent_needed` (``bool``) (mandatory): Whether a user consent is needed during the authentication * :attr:`~eodag.config.PluginConfig.token_exchange_post_data_method` (``str``) (**mandatory**): One of: ``json``, ``data`` or ``params``. This is the way to pass the data to the From 57fd605f8c4e615658beab642fc740dfde0a9a6f Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Thu, 3 Oct 2024 10:45:21 +0200 Subject: [PATCH 18/25] docs: adaptions for plugin config doc --- eodag/config.py | 18 +++++++++- eodag/plugins/authentication/generic.py | 3 +- eodag/plugins/authentication/header.py | 6 ++-- eodag/plugins/search/build_search_result.py | 11 +++--- eodag/plugins/search/cop_marine.py | 7 ++-- eodag/plugins/search/creodias_s3.py | 7 ++-- eodag/plugins/search/csw.py | 10 +++--- eodag/plugins/search/data_request_search.py | 22 ++++++------ eodag/plugins/search/qssearch.py | 38 +++++++++++---------- eodag/plugins/search/static_stac_search.py | 7 ++-- 10 files changed, 78 insertions(+), 51 deletions(-) diff --git a/eodag/config.py b/eodag/config.py index b2aaa7cbc..8969a9371 100644 --- a/eodag/config.py +++ b/eodag/config.py @@ -446,6 +446,9 @@ class MetadataPreMapping(TypedDict, total=False): #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` #: URL to fetch the search result when the data request is done result_url: str + #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` + #: if date parameters are mandatory in the request + dates_required: bool #: :class:`~eodag.plugins.search.csw.CSWSearch` Search definition dictionary search_definition: Dict[str, Any] #: :class:`~eodag.plugins.search.qssearch.PostJsonSearch` Whether to merge responses or not (`aws_eos` specific) @@ -461,6 +464,12 @@ class MetadataPreMapping(TypedDict, total=False): #: :class:`~eodag.plugins.search.build_search_result.BuildSearchResult` #: List of parameters used to parse metadata but that must not be included to the query remove_from_query: List[str] + #: :class:`~eodag.plugins.search.creodias_s3.CreodiasS3Search` + #: base url of the S3 used + s3_endpoint: str + #: :class:`~eodag.plugins.search.csw.CSWSearch` + #: OGC Catalogue Service version + version: str # download --------------------------------------------------------------------------------------------------------- #: :class:`~eodag.plugins.download.base.Download` Default endpoint url @@ -569,8 +578,12 @@ class MetadataPreMapping(TypedDict, total=False): #: :class:`~eodag.plugins.authentication.token.TokenAuth` #: Credentials json structure if they should be sent as POST data req_data: Dict[str, Any] - #: :class:`~eodag.plugins.authentication.token.TokenAuth` URL used to fetch the access token with a refresh token + #: :class:`~eodag.plugins.authentication.token.TokenAuth` + #: URL used to fetch the access token with a refresh token refresh_uri: str + #: :class:`~eodag.plugins.authentication.token.TokenAuth` + #: type of the token + token_type: str #: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth` #: The full :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` plugin configuration #: used to retrieve subject token @@ -581,6 +594,9 @@ class MetadataPreMapping(TypedDict, total=False): #: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth` #: Audience that the token ID is intended for. :attr:`~eodag.config.PluginConfig.client_id` of the Relying Party audience: str + #: :class:`~eodag.plugins.authentication.generic.GenericAuth` + #: which authentication method should be used + method: str yaml_loader = yaml.Loader yaml_dumper = yaml.SafeDumper diff --git a/eodag/plugins/authentication/generic.py b/eodag/plugins/authentication/generic.py index 6799f9964..8df68b1ff 100644 --- a/eodag/plugins/authentication/generic.py +++ b/eodag/plugins/authentication/generic.py @@ -29,7 +29,8 @@ class GenericAuth(Authentication): - """GenericAuth authentication plugin (authentication using username and password) + """GenericAuth authentication plugin (authentication using ``username`` and ``password``) + The mandatory parameters that have to be added in the eodag config are username and password. :param provider: provider name diff --git a/eodag/plugins/authentication/header.py b/eodag/plugins/authentication/header.py index 164838d30..6e64d8291 100644 --- a/eodag/plugins/authentication/header.py +++ b/eodag/plugins/authentication/header.py @@ -54,9 +54,9 @@ class HTTPHeaderAuth(Authentication): ... ... - As you can see in the sample above, the maintainer of `provider` define the headers that will be used in the - authentication process as-is, by giving their names (e.g. `Authorization`) and their value (e.g - `"Something {userinput}"`) as regular Python string templates that enable passing in the user input necessary to + As you can see in the sample above, the maintainer of ``provider`` define the headers that will be used in the + authentication process as-is, by giving their names (e.g. ``Authorization``) and their value (e.g + ``"Something {userinput}"``) as regular Python string templates that enable passing in the user input necessary to compute its identity. The user input awaited in the header value string must be present in the user config file. In the sample above, the plugin await for user credentials to be specified as:: diff --git a/eodag/plugins/search/build_search_result.py b/eodag/plugins/search/build_search_result.py index accfc81db..4dca45ef3 100644 --- a/eodag/plugins/search/build_search_result.py +++ b/eodag/plugins/search/build_search_result.py @@ -81,8 +81,9 @@ class BuildPostSearchResult(PostJsonSearch): performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult` object. - The available configuration parameters are inherited from parent classes (PostJsonSearch and - QueryStringSearch), with particularly for this plugin: + The available configuration parameters are inherited from parent classes + (:class:`~eodag.plugins.search.qssearch.PostJsonSearch` and + :class:`~eodag.plugins.search.qssearch.QueryStringSearch`), with particularly for this plugin: :param provider: provider name :param config: Search plugin configuration: @@ -243,8 +244,10 @@ class BuildSearchResult(BuildPostSearchResult): This plugin builds a single :class:`~eodag.api.search_result.SearchResult` object using given query parameters as product properties. - The available configuration parameters inherits from parent classes (BuildPostSearchResult, - PostJsonSearch and QueryStringSearch), with particularly for this plugin: + The available configuration parameters inherits from parent classes + (:class:`~eodag.plugins.search.build_search_result.BuildPostSearchResult`, + :class:`~eodag.plugins.search.qssearch.PostJsonSearch` and + :class:`~eodag.plugins.search.qssearch.QueryStringSearch`), with particularly for this plugin: :param provider: provider name :param config: Search plugin configuration: diff --git a/eodag/plugins/search/cop_marine.py b/eodag/plugins/search/cop_marine.py index 28124c2bc..871c5b35a 100644 --- a/eodag/plugins/search/cop_marine.py +++ b/eodag/plugins/search/cop_marine.py @@ -110,9 +110,12 @@ def _check_int_values_properties(properties: Dict[str, Any]): class CopMarineSearch(StaticStacSearch): """class that implements search for the Copernicus Marine provider - It calls discover_product_types inherited from StaticStacSearch but for the actual search - a special method which fetches the urls of the available products from an S3 storage and + + It calls :meth:`~eodag.plugins.search.static_stac_search.StaticStacSearch.discover_product_types` + inherited from :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch` + but for the actual search a special method which fetches the urls of the available products from an S3 storage and filters them has been written. + The configuration parameters are inherited from the parent and grand-parent classes. The :attr:`~eodag.config.PluginConfig.DiscoverMetadata.auto_discovery` parameter in the :attr:`~eodag.config.PluginConfig.discover_metadata` section has to be set to ``false`` and the diff --git a/eodag/plugins/search/creodias_s3.py b/eodag/plugins/search/creodias_s3.py index 3c34f61d0..905483ee8 100644 --- a/eodag/plugins/search/creodias_s3.py +++ b/eodag/plugins/search/creodias_s3.py @@ -117,9 +117,10 @@ def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth): class CreodiasS3Search(ODataV4Search): """ - CreodiasS3Search is an extension of ODataV4Search, it executes a Search on creodias and - adapts results so that the assets contain links to s3. It has the same configuration - parameters as ODataV4Search and one additional parameter: + ``CreodiasS3Search`` is an extension of :class:`~eodag.plugins.search.qssearch.ODataV4Search`, + it executes a Search on creodias and adapts results so that the assets contain links to s3. + It has the same configuration parameters as :class:`~eodag.plugins.search.qssearch.ODataV4Search` and + one additional parameter: :param provider: provider name :param config: Search plugin configuration: diff --git a/eodag/plugins/search/csw.py b/eodag/plugins/search/csw.py index c67cedbef..996519825 100644 --- a/eodag/plugins/search/csw.py +++ b/eodag/plugins/search/csw.py @@ -62,9 +62,9 @@ class CSWSearch(Search): * :attr:`~eodag.config.PluginConfig.version` (``str``): OGC Catalogue Service version; default: ``2.0.2`` * :attr:`~eodag.config.PluginConfig.search_definition` (``Dict[str, Any]``) (**mandatory**): - * **product_type_tags** (``List[Dict[str, Any]``): dict of product type tags - * **resource_location_filter** (``str``): regex string - * **date_tags** (``Dict[str, Any]``): tags for start and end + * **product_type_tags** (``List[Dict[str, Any]``): dict of product type tags + * **resource_location_filter** (``str``): regex string + * **date_tags** (``Dict[str, Any]``): tags for start and end * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Any]``): The search plugins of this kind can detect when a metadata mapping is "query-able", and get the semantics of how to format the query string @@ -78,8 +78,8 @@ class CSWSearch(Search): - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}' - '$.properties.acquisition.endViewingDate' - means that the search url will have a query string parameter named *"f"* with a value of - *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done with the value + means that the search url will have a query string parameter named ``f`` with a value of + ``acquisition.endViewingDate:lte:1543922280.0`` if the search was done with the value of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value of ``completionTimeFromAscendingNode``. This example shows all there is to know about the diff --git a/eodag/plugins/search/data_request_search.py b/eodag/plugins/search/data_request_search.py index d1d67b836..ced1eab56 100644 --- a/eodag/plugins/search/data_request_search.py +++ b/eodag/plugins/search/data_request_search.py @@ -90,15 +90,15 @@ class DataRequestSearch(Search): (**mandatory**): The configuration of how the pagination is done on the provider. It is a tree with the following nodes: - * :attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path` (``str``): An XPath or JsonPath - leading to the total number of results satisfying a request. This is used for providers which provides the - total results metadata along with the result of the query and don't have an endpoint for querying - the number of items satisfying a request, or for providers for which the count endpoint - returns a json or xml document - * :attr:`~eodag.config.PluginConfig.Pagination.max_items_per_page` (``int``): The maximum - number of items per page that the provider can handle; default: ``50`` - * :attr:`~eodag.config.PluginConfig.Pagination.start_page` (``int``): number of the - first page; default: ``1`` + * :attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path` (``str``): An XPath or JsonPath + leading to the total number of results satisfying a request. This is used for providers which provides the + total results metadata along with the result of the query and don't have an endpoint for querying + the number of items satisfying a request, or for providers for which the count endpoint + returns a json or xml document + * :attr:`~eodag.config.PluginConfig.Pagination.max_items_per_page` (``int``): The maximum + number of items per page that the provider can handle; default: ``50`` + * :attr:`~eodag.config.PluginConfig.Pagination.start_page` (``int``): number of the + first page; default: ``1`` * :attr:`~eodag.config.PluginConfig.discover_product_types` (:class:`~eodag.config.PluginConfig.DiscoverProductTypes`): configuration for product type discovery based on @@ -145,8 +145,8 @@ class DataRequestSearch(Search): - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}' - '$.properties.acquisition.endViewingDate' - means that the search url will have a query string parameter named *"f"* with a value of - *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done with the value + means that the search url will have a query string parameter named ``f`` with a value of + ``acquisition.endViewingDate:lte:1543922280.0`` if the search was done with the value of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value of ``completionTimeFromAscendingNode``. This example shows all there is to know about the diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index 2fa0bc03f..4528562fd 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -224,8 +224,8 @@ class QueryStringSearch(Search): - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}' - '$.properties.acquisition.endViewingDate' - means that the search url will have a query string parameter named *"f"* with a value of - *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done with the value + means that the search url will have a query string parameter named ``f`` with a value of + ``acquisition.endViewingDate:lte:1543922280.0`` if the search was done with the value of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value of ``completionTimeFromAscendingNode``. This example shows all there is to know about the @@ -1388,26 +1388,27 @@ def normalize_results( class PostJsonSearch(QueryStringSearch): - """A specialisation of a QueryStringSearch that uses POST method - All configuration parameters available for QueryStringSearch are also available for PostJsonSearch. - The mappings given in metadata_mapping are used to construct a (json) body for the - POST request that is sent to the provider. Due to the fact that we sent a POST request and + """A specialisation of a :class:`~eodag.plugins.search.qssearch.QueryStringSearch` that uses POST method + + All configuration parameters available for :class:`~eodag.plugins.search.qssearch.QueryStringSearch` + are also available for PostJsonSearch. The mappings given in metadata_mapping are used to construct + a (json) body for the POST request that is sent to the provider. Due to the fact that we sent a POST request and not a get request, the pagination configuration will look slightly different. It has the following parameters: :param provider: provider name :param config: Search plugin configuration: - * :attr:`~eodag.config.PluginConfig.next_page_query_obj` (``str``): The additional parameters + * :attr:`~eodag.config.PluginConfig.Pagination.next_page_query_obj` (``str``): The additional parameters needed to add pagination information to the search request. These parameters won't be - included in result. This must be a json dict formatted like `{{"foo":"bar"}}` because - it will be passed to a `.format()` method before being loaded as json. - * :attr:`~eodag.config.PluginConfig.total_items_nb_key_path*` (``str``): An XPath or JsonPath + included in result. This must be a json dict formatted like ``{{"foo":"bar"}}`` because + it will be passed to a :meth:`str.format` method before being loaded as json. + * :attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path*` (``str``): An XPath or JsonPath leading to the total number of results satisfying a request. This is used for providers which provides the total results metadata along with the result of the query and don't have an endpoint for querying the number of items satisfying a request, or for providers for which the count endpoint returns a json or xml document - * :attr:`~eodag.config.PluginConfig.max_items_per_page` (``int``): The maximum number of items + * :attr:`~eodag.config.PluginConfig.Pagination.max_items_per_page` (``int``): The maximum number of items per page that the provider can handle; default: ``50`` """ @@ -1794,12 +1795,13 @@ def _request( class StacSearch(PostJsonSearch): - """A specialisation of PostJsonSearch that uses generic STAC configuration, it therefore - has the same configuration parameters (those inherited from QueryStringSearch) - For providers using StacSearch default values are defined for most of the parameters - (see stac_provider.yml). If some parameters are different for a specific provider, they + """A specialisation of :class:`~eodag.plugins.search.qssearch.PostJsonSearch` that uses generic + STAC configuration, it therefore has the same configuration parameters (those inherited + from :class:`~eodag.plugins.search.qssearch.QueryStringSearch`). + For providers using ``StacSearch`` default values are defined for most of the parameters + (see ``stac_provider.yml``). If some parameters are different for a specific provider, they have to be overwritten. If certain functionalities are not available, their configuration - parameters have to be overwritten with `null`. E.g. if there is no queryables endpoint, + parameters have to be overwritten with ``null``. E.g. if there is no queryables endpoint, the :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` and :attr:`~eodag.config.PluginConfig.DiscoverQueryables.product_type_fetch_url` in the :attr:`~eodag.config.PluginConfig.discover_queryables` config have to be set to ``null``. @@ -1959,8 +1961,8 @@ def discover_queryables( class PostJsonSearchWithStacQueryables(StacSearch, PostJsonSearch): - """A specialisation of a :class:`~eodag.plugins.search.qssearch.PostJsonSearch` that - uses generic STAC configuration for queryables (inherited from StacSearch). + """A specialisation of a :class:`~eodag.plugins.search.qssearch.PostJsonSearch` that uses + generic STAC configuration for queryables (inherited from :class:`~eodag.plugins.search.qssearch.StacSearch`). """ def __init__(self, provider: str, config: PluginConfig) -> None: diff --git a/eodag/plugins/search/static_stac_search.py b/eodag/plugins/search/static_stac_search.py index 49e52771e..c1f9fdbd0 100644 --- a/eodag/plugins/search/static_stac_search.py +++ b/eodag/plugins/search/static_stac_search.py @@ -43,12 +43,13 @@ class StaticStacSearch(StacSearch): """Static STAC Catalog search plugin + This plugin first loads all STAC items found in the catalog, and converts them to - EOProducts using StacSearch. + EOProducts using :class:`~eodag.plugins.search.qssearch.StacSearch`. Then it uses crunchers to only keep products matching query parameters. - The plugin inherits the configuration parameters from PostJsonSearch (via the StacSearch - inheritance) with the following particularities: + The plugin inherits the configuration parameters from :class:`~eodag.plugins.search.qssearch.PostJsonSearch` + (via the :class:`~eodag.plugins.search.qssearch.StacSearch` inheritance) with the following particularities: :param provider: provider name :param config: Search plugin configuration: From e7777e47bddc4bd912b1a09d4b2513af4de19053 Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Thu, 3 Oct 2024 10:59:48 +0200 Subject: [PATCH 19/25] fix: remove duplicate in plugin config --- eodag/config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/eodag/config.py b/eodag/config.py index 8969a9371..1c0094395 100644 --- a/eodag/config.py +++ b/eodag/config.py @@ -464,9 +464,6 @@ class MetadataPreMapping(TypedDict, total=False): #: :class:`~eodag.plugins.search.build_search_result.BuildSearchResult` #: List of parameters used to parse metadata but that must not be included to the query remove_from_query: List[str] - #: :class:`~eodag.plugins.search.creodias_s3.CreodiasS3Search` - #: base url of the S3 used - s3_endpoint: str #: :class:`~eodag.plugins.search.csw.CSWSearch` #: OGC Catalogue Service version version: str From 6a33c06968530c1178f60bc25ba763e000d7f633 Mon Sep 17 00:00:00 2001 From: jlahovnik Date: Wed, 16 Oct 2024 12:07:34 +0200 Subject: [PATCH 20/25] docs: fixes for plugin config docstrings --- eodag/config.py | 15 +++++++++++++++ eodag/plugins/apis/base.py | 14 ++++++++++---- eodag/plugins/apis/usgs.py | 5 +++-- eodag/plugins/authentication/aws_auth.py | 2 +- eodag/plugins/authentication/keycloak.py | 8 ++++---- eodag/plugins/authentication/oauth.py | 3 ++- eodag/plugins/authentication/openid_connect.py | 6 ++++-- eodag/plugins/authentication/qsauth.py | 6 +++--- eodag/plugins/authentication/sas_auth.py | 17 ++++++++++------- eodag/plugins/download/aws.py | 12 ++++++------ eodag/plugins/download/creodias_s3.py | 2 +- eodag/plugins/download/s3rest.py | 11 ++++++----- 12 files changed, 65 insertions(+), 36 deletions(-) diff --git a/eodag/config.py b/eodag/config.py index 1c0094395..f3fd1c820 100644 --- a/eodag/config.py +++ b/eodag/config.py @@ -551,6 +551,9 @@ class MetadataPreMapping(TypedDict, total=False): authentication_uri_source: str #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` #: The callback url that will handle the code given by the OIDC provider + authentication_uri: str + #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` + #: The URL of the authentication backend of the OIDC provider redirect_uri: str #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` #: The authorization url of the server (where to query for grants) @@ -565,6 +568,18 @@ class MetadataPreMapping(TypedDict, total=False): #: The data that will be passed with the POST request on the form 'action' URL user_consent_form_data: Dict[str, str] #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` + #: Additional data to be passed to the login POST request + additional_login_form_data: Dict[str, str] + #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` + #: Key/value pairs of patterns/messages used for Authentication errors + exchange_url_error_pattern: Dict[str, str] + #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` + #: A mapping between OIDC url query string and token handler query string params + token_exchange_params: Dict[str, str] + #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` + #: Refers to the name of the query param to be used in the query request + token_qs_key: str + #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` #: Way to pass the data to the POST request that is made to the token server token_exchange_post_data_method: str #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` diff --git a/eodag/plugins/apis/base.py b/eodag/plugins/apis/base.py index 766500d93..5ea263e41 100644 --- a/eodag/plugins/apis/base.py +++ b/eodag/plugins/apis/base.py @@ -24,9 +24,10 @@ class Api(Search, Download): """Plugins API Base plugin - An Api plugin inherit the methods from Search and Download plugins. + An Api plugin inherits the methods from Search and Download plugins. There are three methods that it must implement: + - ``query``: search for products - ``download``: download a single :class:`~eodag.api.product._product.EOProduct` - ``download_all``: download multiple products from a :class:`~eodag.api.search_result.SearchResult` @@ -35,14 +36,14 @@ class Api(Search, Download): - download data in the ``output_dir`` folder defined in the plugin's configuration or passed through kwargs - - extract products from their archive (if relevant) if ``extract`` is set to True - (True by default) + - extract products from their archive (if relevant) if ``extract`` is set to ``True`` + (``True`` by default) - save a product in an archive/directory (in ``output_dir``) whose name must be the product's ``title`` property - update the product's ``location`` attribute once its data is downloaded (and eventually after it's extracted) to the product's location given as a file URI (e.g. 'file:///tmp/product_folder' on Linux or - 'file:///C:/Users/username/AppData/LOcal/Temp' on Windows) + 'file:///C:/Users/username/AppData/Local/Temp' on Windows) - save a *record* file in the directory ``output_dir/.downloaded`` whose name is built on the MD5 hash of the product's ``product_type`` and ``properties['id']`` attributes (``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``) @@ -52,4 +53,9 @@ class Api(Search, Download): - not try to download a product if its *record* file exists as long as the expected product's file/directory. If the *record* file only is found, it must be deleted (it certainly indicates that the download didn't complete) + + :param provider: An EODAG provider name + :type provider: str + :param config: An EODAG plugin configuration + :type config: Dict[str, Any] """ diff --git a/eodag/plugins/apis/usgs.py b/eodag/plugins/apis/usgs.py index b869ff3c3..79bf49bdf 100644 --- a/eodag/plugins/apis/usgs.py +++ b/eodag/plugins/apis/usgs.py @@ -74,8 +74,9 @@ class UsgsApi(Api): :param config: Api plugin configuration: * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): UsgsApi - * :attr:`~eodag.config.PluginConfig.pagination` (``Dict[str, Any]``) (**mandatory**): dict - containing parameters for pagination; should contain the key ``total_items_nb_key_path`` + * :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`) + (**mandatory**): object containing parameters for pagination; should contain the attribute + :attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path` which is indicating the key for the number of total items in the provider result * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in the download request; default: ``True`` diff --git a/eodag/plugins/authentication/aws_auth.py b/eodag/plugins/authentication/aws_auth.py index df84f436a..85ef36b6a 100644 --- a/eodag/plugins/authentication/aws_auth.py +++ b/eodag/plugins/authentication/aws_auth.py @@ -44,7 +44,7 @@ class AwsAuth(Authentication): :param config: Authentication plugin configuration: * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): AwsAuth - * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): (mandatory for creodias_s3) + * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``) (mandatory for ``creodias_s3``): which error code is returned in case of an authentication error """ diff --git a/eodag/plugins/authentication/keycloak.py b/eodag/plugins/authentication/keycloak.py index a8ec0b172..b2e17db64 100644 --- a/eodag/plugins/authentication/keycloak.py +++ b/eodag/plugins/authentication/keycloak.py @@ -55,7 +55,7 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase): client secret, set to null if no secret is used * :attr:`~eodag.config.PluginConfig.token_provision` (``str``) (**mandatory**): if the token should be added to the query string (``qs``) or to the header (``header``) - * :attr:`~eodag.config.PluginConfig.token_qs_key` (``str``): (mandatory if token_provision=qs) + * :attr:`~eodag.config.PluginConfig.token_qs_key` (``str``): (**mandatory if token_provision=qs**) key of the param added to the query string * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an authentication error @@ -63,8 +63,8 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase): should be verified in the token request; default: ``True`` Using :class:`~eodag.plugins.download.http.HTTPDownload` a download link - `http://example.com?foo=bar` will become - `http://example.com?foo=bar&my-token=obtained-token` if associated to the following + ``http://example.com?foo=bar`` will become + ``http://example.com?foo=bar&my-token=obtained-token`` if associated to the following configuration:: provider: @@ -81,7 +81,7 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase): ... If configured to send the token through the header, the download request header will - be updated with `Authorization: "Bearer obtained-token"` if associated to the + be updated with ``Authorization: "Bearer obtained-token"`` if associated to the following configuration:: provider: diff --git a/eodag/plugins/authentication/oauth.py b/eodag/plugins/authentication/oauth.py index 6c0b2b6b4..1cf90d300 100644 --- a/eodag/plugins/authentication/oauth.py +++ b/eodag/plugins/authentication/oauth.py @@ -28,7 +28,8 @@ class OAuth(Authentication): """OAuth authentication plugin - The mandatory parameters that have to be added in the eodag config are aws_access_key_id and aws_secret_access_key. + The mandatory parameters that have to be added in the eodag config are ``aws_access_key_id`` + and ``aws_secret_access_key``. :param provider: provider name :param config: Authentication plugin configuration: diff --git a/eodag/plugins/authentication/openid_connect.py b/eodag/plugins/authentication/openid_connect.py index 32ceb3e55..4cfc2d520 100644 --- a/eodag/plugins/authentication/openid_connect.py +++ b/eodag/plugins/authentication/openid_connect.py @@ -44,8 +44,9 @@ class OIDCRefreshTokenBase(Authentication): """OIDC refresh token base class, to be used through specific OIDC flows plugins; Common mechanism to handle refresh token from all OIDC auth plugins; - Plugins inheriting from this base class must implement the methods _request_new_token and - _get_token_with_refresh_token. Depending oh the implementation of these methods they can have + + Plugins inheriting from this base class must implement the methods ``_request_new_token()`` and + ``_get_token_with_refresh_token()``. Depending on the implementation of these methods they can have different configuration parameters. """ @@ -174,6 +175,7 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase): adds an authentication layer on top of oauth 2.0. This plugin implements the `authorization code flow `_ option of this specification. + The particularity of this plugin is that it proceeds to a headless (not involving the user) interaction with the OpenID provider (if necessary) to authenticate a registered user with its username and password on the server and then granting to eodag the diff --git a/eodag/plugins/authentication/qsauth.py b/eodag/plugins/authentication/qsauth.py index ab7513263..80367910d 100644 --- a/eodag/plugins/authentication/qsauth.py +++ b/eodag/plugins/authentication/qsauth.py @@ -45,8 +45,8 @@ class HttpQueryStringAuth(Authentication): given in the configuration Using :class:`~eodag.plugins.download.http.HTTPDownload` a download link - `http://example.com?foo=bar` will become - `http://example.com?foo=bar&apikey=XXX&otherkey=YYY` if associated to the following + ``http://example.com?foo=bar`` will become + ``http://example.com?foo=bar&apikey=XXX&otherkey=YYY`` if associated to the following configuration:: provider: @@ -64,7 +64,7 @@ class HttpQueryStringAuth(Authentication): ... ... - If `auth_uri` is specified (optional), it will be used to check credentials through + If ``auth_uri`` is specified (optional), it will be used to check credentials through :meth:`~eodag.plugins.authentication.query_string.HttpQueryStringAuth.authenticate` """ diff --git a/eodag/plugins/authentication/sas_auth.py b/eodag/plugins/authentication/sas_auth.py index 672ead262..4b0e9cb98 100644 --- a/eodag/plugins/authentication/sas_auth.py +++ b/eodag/plugins/authentication/sas_auth.py @@ -86,18 +86,21 @@ def __call__(self, request: PreparedRequest) -> PreparedRequest: class SASAuth(Authentication): """SASAuth authentication plugin + An apiKey that is added in the headers can be given in the credentials in the config file. :param provider: provider name :param config: Authentication plugin configuration: - * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): SASAuth - * :attr:`~eodag.config.PluginConfig.auth_uri` (``str``) (**mandatory**): url used to get the signed url - * :attr:`~eodag.config.PluginConfig.signed_url_key (``str``) (**mandatory**): key to get the signed url - * :attr:`~eodag.config.PluginConfig.headers** (``Dict[str, str]``): (**mandatory if apiKey is used**) - headers to be added to the requests - * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be - verified in the requests; default: True + * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): SASAuth + * :attr:`~eodag.config.PluginConfig.auth_uri` (``str``) (**mandatory**): url used to + get the signed url + * :attr:`~eodag.config.PluginConfig.signed_url_key` (``str``) (**mandatory**): key to + get the signed url + * :attr:`~eodag.config.PluginConfig.headers` (``Dict[str, str]``) (**mandatory if + apiKey is used**): headers to be added to the requests + * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be + verified in the requests; default: ``True`` """ diff --git a/eodag/plugins/download/aws.py b/eodag/plugins/download/aws.py index ee13568c0..f6924124e 100644 --- a/eodag/plugins/download/aws.py +++ b/eodag/plugins/download/aws.py @@ -224,7 +224,7 @@ class AwsDownload(Download): * :attr:`~eodag.config.PluginConfig.flatten_top_dirs` (``bool``): if the directory structure should be flattened; default: ``True`` * :attr:`~eodag.config.PluginConfig.ignore_assets` (``bool``): ignore assets and download - using downloadLink; default: ``False`` + using ``downloadLink``; default: ``False`` * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in requests; default: ``True`` * :attr:`~eodag.config.PluginConfig.bucket_path_level` (``int``): at which level of the @@ -233,11 +233,11 @@ class AwsDownload(Download): * :attr:`~eodag.config.PluginConfig.products` (``Dict[str, Dict[str, Any]``): product type specific config; the keys are the product types, the values are dictionaries which can contain the keys: - * **default_bucket** (``str``): bucket where the product type can be found - * **complementary_url_key** (``str``): keys to add additional urls - * **build_safe** (``bool``): if a SAFE (Standard Archive Format for Europe) product should - be created; used for Sentinel products; default: False - * **fetch_metadata** (``Dict[str, Any]``): config for metadata to be fetched for the SAFE product + * **default_bucket** (``str``): bucket where the product type can be found + * **complementary_url_key** (``str``): keys to add additional urls + * **build_safe** (``bool``): if a SAFE (Standard Archive Format for Europe) product should + be created; used for Sentinel products; default: False + * **fetch_metadata** (``Dict[str, Any]``): config for metadata to be fetched for the SAFE product """ diff --git a/eodag/plugins/download/creodias_s3.py b/eodag/plugins/download/creodias_s3.py index 5ee504543..735984e2c 100644 --- a/eodag/plugins/download/creodias_s3.py +++ b/eodag/plugins/download/creodias_s3.py @@ -25,7 +25,7 @@ class CreodiasS3Download(AwsDownload): """ - Download on creodias s3 from their VMs (extension of AwsDownload) + Download on creodias s3 from their VMs (extension of :class:`~eodag.plugins.download.aws.AwsDownload`) :param provider: provider name :param config: Download plugin configuration: diff --git a/eodag/plugins/download/s3rest.py b/eodag/plugins/download/s3rest.py index 6141d6742..1158a70e8 100644 --- a/eodag/plugins/download/s3rest.py +++ b/eodag/plugins/download/s3rest.py @@ -62,10 +62,11 @@ class S3RestDownload(Download): """Http download on S3-like object storage location - for example using Mundi REST API (free account) + + For example using Mundi REST API (free account) https://mundiwebservices.com/keystoneapi/uploads/documents/CWS-DATA-MUT-087-EN-Mundi_Download_v1.1.pdf#page=13 - Re-use AwsDownload bucket some handling methods + Re-use AwsDownload bucket and some handling methods :param provider: provider name :param config: Download plugin configuration: @@ -73,14 +74,14 @@ class S3RestDownload(Download): * :attr:`~eodag.config.PluginConfig.base_uri` (``str``) (**mandatory**): default endpoint url * :attr:`~eodag.config.PluginConfig.extract` (``bool``): extract downloaded archive or not * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): authentication error code - * :attr:`~eodag.config.PluginConfig.bucket_path_level` (``int``): bucket location index in path.split('/') + * :attr:`~eodag.config.PluginConfig.bucket_path_level` (``int``): bucket location index in ``path.split('/')`` * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): whether order is enabled or not if product is `OFFLINE` * :attr:`~eodag.config.PluginConfig.order_method` (``str``) HTTP request method, ``GET`` (default) or ``POST`` * :attr:`~eodag.config.PluginConfig.order_headers` (``[Dict[str, str]]``): order request headers * :attr:`~eodag.config.PluginConfig.order_on_response` (:class:`~eodag.config.PluginConfig.OrderOnResponse`): - a typed dictionary containing the key 'metadata_mapping' which can be used to add new product properties - based on the data in response to the order request + a typed dictionary containing the key :attr:`~eodag.config.PluginConfig.OrderOnResponse.metadata_mapping` + which can be used to add new product properties based on the data in response to the order request * :attr:`~eodag.config.PluginConfig.order_status` (:class:`~eodag.config.PluginConfig.OrderStatus`): Order status handling """ From b6ed38b04839fc429c9eedd7a9cadf87f8d60dad Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Mon, 21 Oct 2024 15:44:03 +0200 Subject: [PATCH 21/25] docs: base search parameters quotes --- eodag/plugins/search/base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eodag/plugins/search/base.py b/eodag/plugins/search/base.py index 29d489c71..2d6956115 100644 --- a/eodag/plugins/search/base.py +++ b/eodag/plugins/search/base.py @@ -215,10 +215,10 @@ def get_metadata_mapping( return self.config.metadata_mapping def get_sort_by_arg(self, kwargs: Dict[str, Any]) -> Optional[SortByList]: - """Extract the "sort_by" argument from the kwargs or the provider default sort configuration + """Extract the ``sort_by`` argument from the kwargs or the provider default sort configuration :param kwargs: Search arguments - :returns: The "sort_by" argument from the kwargs or the provider default sort configuration + :returns: The ``sort_by`` argument from the kwargs or the provider default sort configuration """ # remove "sort_by" from search args if exists because it is not part of metadata mapping, # it will complete the query string or body once metadata mapping will be done @@ -237,10 +237,10 @@ def build_sort_by( self, sort_by_arg: SortByList ) -> Tuple[str, Dict[str, List[Dict[str, str]]]]: """Build the sorting part of the query string or body by transforming - the "sort_by" argument into a provider-specific string or dictionary + the ``sort_by`` argument into a provider-specific string or dictionary - :param sort_by_arg: the "sort_by" argument in EODAG format - :returns: The "sort_by" argument in provider-specific format + :param sort_by_arg: the ``sort_by`` argument in EODAG format + :returns: The ``sort_by`` argument in provider-specific format """ if not hasattr(self.config, "sort"): raise ValidationError(f"{self.provider} does not support sorting feature") From 87604cf5d4f33b8697968914e9f2f9c20417d48c Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Mon, 21 Oct 2024 15:44:21 +0200 Subject: [PATCH 22/25] docs: docstring formatting fixes --- eodag/plugins/authentication/openid_connect.py | 6 +++--- eodag/plugins/search/qssearch.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eodag/plugins/authentication/openid_connect.py b/eodag/plugins/authentication/openid_connect.py index 4cfc2d520..332eb2786 100644 --- a/eodag/plugins/authentication/openid_connect.py +++ b/eodag/plugins/authentication/openid_connect.py @@ -214,8 +214,8 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase): to look for the authentication_uri. One of ``config`` (in the configuration) or ``login-form`` (use the 'action' URL found in the login form retrieved with login_form_xpath). If the value is ``config``, authentication_uri config param is mandatory - * :attr:`~eodag.config.PluginConfig.authentication_uri` (``str``): (mandatory if - authentication_uri_source=config) The URL of the authentication backend of the OIDC provider + * :attr:`~eodag.config.PluginConfig.authentication_uri` (``str``): (**mandatory if + authentication_uri_source=config**) The URL of the authentication backend of the OIDC provider * :attr:`~eodag.config.PluginConfig.user_consent_form_xpath` (``str``): The xpath to the user consent form. The form is searched in the content of the response to the authorization request * :attr:`~eodag.config.PluginConfig.user_consent_form_data` (``Dict[str, str]``): The data that @@ -223,7 +223,7 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase): key value pairs, the keys representing the data key and the value being either a 'constant' string value, or a string of the form 'xpath()' and representing a value to be retrieved in the user consent form. The xpath must resolve directly to a - string value, not to an HTML element. Example: `xpath(//input[@name="sessionDataKeyConsent"]/@value)` + string value, not to an HTML element. Example: ``xpath(//input[@name="sessionDataKeyConsent"]/@value)`` * :attr:`~eodag.config.PluginConfig.additional_login_form_data` (``Dict[str, str]``): A mapping giving additional data to be passed to the login POST request. The value follows the same rules as with user_consent_form_data diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index 4528562fd..a3b93f869 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -1403,7 +1403,7 @@ class PostJsonSearch(QueryStringSearch): needed to add pagination information to the search request. These parameters won't be included in result. This must be a json dict formatted like ``{{"foo":"bar"}}`` because it will be passed to a :meth:`str.format` method before being loaded as json. - * :attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path*` (``str``): An XPath or JsonPath + * :attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path` (``str``): An XPath or JsonPath leading to the total number of results satisfying a request. This is used for providers which provides the total results metadata along with the result of the query and don't have an endpoint for querying the number of items satisfying a request, or for providers From 8ae300bbb0c6a61220d08197eb6900bde63c1747 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Mon, 21 Oct 2024 15:44:37 +0200 Subject: [PATCH 23/25] docs: crunch plugins --- eodag/plugins/crunch/base.py | 2 +- eodag/plugins/crunch/filter_date.py | 9 +++++---- eodag/plugins/crunch/filter_latest_intersect.py | 6 +++--- eodag/plugins/crunch/filter_latest_tpl_name.py | 6 ++---- eodag/plugins/crunch/filter_overlap.py | 17 +++++------------ eodag/plugins/crunch/filter_property.py | 9 ++++----- eodag/plugins/search/base.py | 14 ++++++-------- 7 files changed, 26 insertions(+), 37 deletions(-) diff --git a/eodag/plugins/crunch/base.py b/eodag/plugins/crunch/base.py index 5bb367c32..15d042b63 100644 --- a/eodag/plugins/crunch/base.py +++ b/eodag/plugins/crunch/base.py @@ -28,8 +28,8 @@ class Crunch(PluginTopic): """Base cruncher + :param config: Crunch configuration - :type config: Dict[str, Any] """ def __init__(self, config: Optional[Dict[str, Any]]) -> None: diff --git a/eodag/plugins/crunch/filter_date.py b/eodag/plugins/crunch/filter_date.py index dc67bca9b..0a77c5399 100644 --- a/eodag/plugins/crunch/filter_date.py +++ b/eodag/plugins/crunch/filter_date.py @@ -37,12 +37,13 @@ class FilterDate(Crunch): """FilterDate cruncher: filter products by date - The Crunch configuration, may contain : + Allows to filter out products that are older than a start date (optional) or more recent than an end date + (optional). - :param config: Search plugin configuration: + :param config: Crunch configuration, may contain : - * :attr:`~eodag.config.PluginConfig.start` (``str``): start sensing time in iso format - * :attr:`~eodag.config.PluginConfig.end` (``str``): end sensing time in iso format + * ``start`` (``str``): start sensing time in iso format + * ``end`` (``str``): end sensing time in iso format """ @staticmethod diff --git a/eodag/plugins/crunch/filter_latest_intersect.py b/eodag/plugins/crunch/filter_latest_intersect.py index aaa404f77..09abdbd05 100644 --- a/eodag/plugins/crunch/filter_latest_intersect.py +++ b/eodag/plugins/crunch/filter_latest_intersect.py @@ -54,14 +54,14 @@ def sort_product_by_start_date(product: EOProduct) -> dt: return dateutil.parser.parse(start_date) def proceed( - self, products: List[EOProduct], **search_params: Any + self, products: List[EOProduct], **search_params: Dict[str, Any] ) -> List[EOProduct]: """Execute crunch: Filter latest products (the ones with a the highest start date) that intersect search extent. :param products: A list of products resulting from a search - :param search_params: Search criteria that must contain `geometry` (dict) - or search `geom` (:class:`shapely.geometry.base.BaseGeometry`) argument will be used + :param search_params: Search criteria that must contain ``geometry`` or ``geom`` parameters having value of + type :class:`shapely.geometry.base.BaseGeometry` or ``Dict[str, Any]`` :returns: The filtered products """ logger.debug("Start filtering for latest products") diff --git a/eodag/plugins/crunch/filter_latest_tpl_name.py b/eodag/plugins/crunch/filter_latest_tpl_name.py index a98f0821d..2acaeb5fb 100644 --- a/eodag/plugins/crunch/filter_latest_tpl_name.py +++ b/eodag/plugins/crunch/filter_latest_tpl_name.py @@ -35,11 +35,9 @@ class FilterLatestByName(Crunch): Filter Search results to get only the latest product, based on the name of the product - The Crunch configuration must contain : + :param config: Crunch configuration, must contain : - :param config: Search plugin configuration: - - * :attr:`~eodag.config.PluginConfig.name_pattern` (``str``) (**mandatory**): product name pattern + * ``name_pattern`` (``str``) (**mandatory**): product name pattern """ NAME_PATTERN_CONSTRAINT = re.compile(r"\(\?P\\d\{6\}\)") diff --git a/eodag/plugins/crunch/filter_overlap.py b/eodag/plugins/crunch/filter_overlap.py index c321952c4..6446adc80 100644 --- a/eodag/plugins/crunch/filter_overlap.py +++ b/eodag/plugins/crunch/filter_overlap.py @@ -40,19 +40,12 @@ class FilterOverlap(Crunch): Filter products, retaining only those that are overlapping with the search_extent - The Crunch configuration may contain the following parameters which are mutually exclusive: - - :param config: Search plugin configuration: - - * :attr:`~eodag.config.PluginConfig.minimum_overlap` (``Union[float, str]``): minimal overlap - percentage; default: ``"0"`` - * :attr:`~eodag.config.PluginConfig.contains` (``bool``): True if product geometry - contains the search area; default: ``False`` - * :attr:`~eodag.config.PluginConfig.intersects` (``bool``): True if product geometry - intersects the search area; default: ``False`` - * :attr:`~eodag.config.PluginConfig.within` (``bool``): True if product geometry is - within the search area; default: ``False`` + :param config: Crunch configuration may contain the following parameters which are mutually exclusive: + * ``minimum_overlap`` (``Union[float, str]``): minimal overlap percentage; default: ``"0"`` + * ``contains`` (``bool``): ``True`` if product geometry contains the search area; default: ``False`` + * ``intersects`` (``bool``): ``True`` if product geometry intersects the search area; default: ``False`` + * ``within`` (``bool``): ``True`` if product geometry is within the search area; default: ``False`` """ def proceed( diff --git a/eodag/plugins/crunch/filter_property.py b/eodag/plugins/crunch/filter_property.py index ea958fcb4..e699828ea 100644 --- a/eodag/plugins/crunch/filter_property.py +++ b/eodag/plugins/crunch/filter_property.py @@ -34,12 +34,11 @@ class FilterProperty(Crunch): Filter products, retaining only those whose property match criteria - :param config: Search plugin configuration: + :param config: Crunch configuration, must contain : - * ```` ``(Any)`` (**mandatory**): property key from product.properties, - associated to its filter value - * :attr:`~eodag.config.PluginConfig.operator` (``str``): Operator used for filtering - (one of ``lt,le,eq,ne,ge,gt``). Default is ``eq`` + * ```` ``(Any)`` (**mandatory**): property key from ``product.properties``, associated to its filter + value + * ``operator`` (``str``): Operator used for filtering (one of ``lt,le,eq,ne,ge,gt``). Default is ``eq`` """ def proceed( diff --git a/eodag/plugins/search/base.py b/eodag/plugins/search/base.py index 2d6956115..674f89f94 100644 --- a/eodag/plugins/search/base.py +++ b/eodag/plugins/search/base.py @@ -57,9 +57,7 @@ class Search(PluginTopic): """Base Search Plugin. :param provider: An EODAG provider name - :type provider: str :param config: An EODAG plugin configuration - :type config: Dict[str, Any] """ auth: Union[AuthBase, Dict[str, str]] @@ -95,9 +93,9 @@ def query( ) -> Tuple[List[EOProduct], Optional[int]]: """Implementation of how the products must be searched goes here. - This method must return a tuple with (1) a list of EOProduct instances (see eodag.api.product module) - which will be processed by a Download plugin (2) and the total number of products matching - the search criteria. If ``prep.count`` is False, the second element returned must be ``None``. + This method must return a tuple with (1) a list of :class:`~eodag.api.product._product.EOProduct` instances + which will be processed by a :class:`~eodag.plugins.download.base.Download` plugin (2) and the total number of + products matching the search criteria. If ``prep.count`` is False, the second element returned must be ``None``. """ raise NotImplementedError("A Search plugin must implement a method named query") @@ -108,9 +106,9 @@ def discover_product_types(self, **kwargs: Any) -> Optional[Dict[str, Any]]: def discover_queryables( self, **kwargs: Any ) -> Optional[Dict[str, Annotated[Any, FieldInfo]]]: - """Fetch queryables list from provider using `discover_queryables` conf + """Fetch queryables list from provider using :attr:`~eodag.config.PluginConfig.discover_queryables` conf - :param kwargs: additional filters for queryables (`productType` and other search + :param kwargs: additional filters for queryables (``productType`` and other search arguments) :returns: fetched queryable parameters dict """ @@ -184,7 +182,7 @@ def get_product_type_cfg_value(self, key: str, default: Any = None) -> Any: Get the value of a configuration option specific to the current product type. This method retrieves the value of a configuration option from the - `product_type_config` attribute. If the option is not found, the provided + ``product_type_config`` attribute. If the option is not found, the provided default value is returned. :param key: The configuration option key. From 650de1cc2ae33788b45b3e0dd5c9509a069e4efb Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Mon, 21 Oct 2024 15:44:45 +0200 Subject: [PATCH 24/25] docs: donwload base plugin --- eodag/plugins/download/base.py | 18 +++++++++--------- eodag/types/download_args.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/eodag/plugins/download/base.py b/eodag/plugins/download/base.py index e07a5911d..fd78441c0 100644 --- a/eodag/plugins/download/base.py +++ b/eodag/plugins/download/base.py @@ -81,15 +81,15 @@ class Download(PluginTopic): - download data in the ``output_dir`` folder defined in the plugin's configuration or passed through kwargs - - extract products from their archive (if relevant) if ``extract`` is set to True - (True by default) + - extract products from their archive (if relevant) if ``extract`` is set to ``True`` + (``True`` by default) - save a product in an archive/directory (in ``output_dir``) whose name must be the product's ``title`` property - update the product's ``location`` attribute once its data is downloaded (and eventually after it's extracted) to the product's location given as a file URI - (e.g. 'file:///tmp/product_folder' on Linux or - 'file:///C:/Users/username/AppData/LOcal/Temp' on Windows) - - save a *record* file in the directory ``output_dir/.downloaded`` whose name + (e.g. ``file:///tmp/product_folder`` on Linux or + ``file:///C:/Users/username/AppData/Local/Temp`` on Windows) + - save a *record* file in the directory ``{output_dir}/.downloaded`` whose name is built on the MD5 hash of the product's ``product_type`` and ``properties['id']`` attributes (``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``) and whose content is the product's ``remote_location`` attribute itself. @@ -130,8 +130,8 @@ def download( and will override any other values defined in a configuration file or with environment variables. :returns: The absolute path to the downloaded product in the local filesystem - (e.g. '/tmp/product.zip' on Linux or - 'C:\\Users\\username\\AppData\\Local\\Temp\\product.zip' on Windows) + (e.g. ``/tmp/product.zip`` on Linux or + ``C:\\Users\\username\\AppData\\Local\\Temp\\product.zip`` on Windows) """ raise NotImplementedError( "A Download plugin must implement a method named download" @@ -155,8 +155,8 @@ def _stream_download_dict( :param wait: (optional) If download fails, wait time in minutes between two download tries :param timeout: (optional) If download fails, maximum time in minutes before stop retrying to download - :param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool) - and `dl_url_params` (dict) can be provided as additional kwargs + :param kwargs: ``output_dir`` (str), ``extract`` (bool), ``delete_archive`` (bool) + and ``dl_url_params`` (dict) can be provided as additional kwargs and will override any other values defined in a configuration file or with environment variables. :returns: Dictionary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments diff --git a/eodag/types/download_args.py b/eodag/types/download_args.py index 0ead61feb..5fe048ba3 100644 --- a/eodag/types/download_args.py +++ b/eodag/types/download_args.py @@ -23,7 +23,7 @@ class DownloadConf(TypedDict, total=False): """Download configuration - :cvar output_prefix: where to store downloaded products, as an absolute file path + :cvar output_dir: where to store downloaded products, as an absolute file path (Default: local temporary directory) :cvar output_extension: downloaded file extension :cvar extract: whether to extract the downloaded products, only applies to archived products From c7cd7edadc88ffbde6873a036623ede714028626 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Mon, 21 Oct 2024 15:44:57 +0200 Subject: [PATCH 25/25] docs: eodag.utils --- docs/api_reference/utils.rst | 13 ++++- docs/conf.py | 1 + eodag/api/core.py | 2 +- eodag/plugins/download/base.py | 2 +- eodag/utils/__init__.py | 100 +++++++++++++++++---------------- 5 files changed, 67 insertions(+), 51 deletions(-) diff --git a/docs/api_reference/utils.rst b/docs/api_reference/utils.rst index 1c1dc1bd1..5c79f588f 100644 --- a/docs/api_reference/utils.rst +++ b/docs/api_reference/utils.rst @@ -1,4 +1,5 @@ .. module:: eodag.utils + :no-index: ===== Utils @@ -11,9 +12,10 @@ Logging :members: Callbacks ------------------ +--------- -.. autofunction::eodag.api.product.DownloadedCallback +.. autoclass:: eodag.utils.DownloadedCallback + :special-members: __call__ .. autofunction:: eodag.utils.ProgressCallback Notebook @@ -21,3 +23,10 @@ Notebook .. automodule:: eodag.utils.notebook :members: + +Misc +---- + +.. automodule:: eodag.utils + :members: + :exclude-members: DownloadedCallback, ProgressCallback, NotebookProgressCallback, get_progress_callback diff --git a/docs/conf.py b/docs/conf.py index a63acbe1a..2ce389a13 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -249,6 +249,7 @@ # https://github.com/psf/requests/issues/6140#issuecomment-1135071992 "python-requests": ("https://requests.readthedocs.io/en/stable/", None), "shapely": ("https://shapely.readthedocs.io/en/stable/", None), + "click": ("https://click.palletsprojects.com/en/latest/", None), } suppress_warnings = ["misc.copy_overwrite"] diff --git a/eodag/api/core.py b/eodag/api/core.py index b80fe8d63..fa4b1505a 100644 --- a/eodag/api/core.py +++ b/eodag/api/core.py @@ -2047,7 +2047,7 @@ def download_all( :param search_result: A collection of EO products resulting from a search :param downloaded_callback: (optional) A method or a callable object which takes as parameter the ``product``. You can use the base class - :class:`~eodag.api.product.DownloadedCallback` and override + :class:`~eodag.utils.DownloadedCallback` and override its ``__call__`` method. Will be called each time a product finishes downloading :param progress_callback: (optional) A method or a callable object diff --git a/eodag/plugins/download/base.py b/eodag/plugins/download/base.py index fd78441c0..05cb3d8a0 100644 --- a/eodag/plugins/download/base.py +++ b/eodag/plugins/download/base.py @@ -458,7 +458,7 @@ def download_all( :param auth: (optional) authenticated object :param downloaded_callback: (optional) A method or a callable object which takes as parameter the ``product``. You can use the base class - :class:`~eodag.api.product.DownloadedCallback` and override + :class:`~eodag.utils.DownloadedCallback` and override its ``__call__`` method. Will be called each time a product finishes downloading :param progress_callback: (optional) A progress callback diff --git a/eodag/utils/__init__.py b/eodag/utils/__init__.py index 46e144f8b..ebb05bb5a 100644 --- a/eodag/utils/__init__.py +++ b/eodag/utils/__init__.py @@ -105,7 +105,7 @@ if TYPE_CHECKING: from jsonpath_ng import JSONPath - from eodag.api.product import EOProduct + from eodag.api.product._product import EOProduct logger = py_logging.getLogger("eodag.utils") @@ -232,9 +232,10 @@ def __repr__(self) -> str: def slugify(value: Any, allow_unicode: bool = False) -> str: """Copied from Django Source code, only modifying last line (no need for safe strings). + source: https://github.com/django/django/blob/master/django/utils/text.py - Convert to ASCII if 'allow_unicode' is False. Convert spaces to hyphens. + Convert to ASCII if ``allow_unicode`` is ``False``. Convert spaces to hyphens. Remove characters that aren't alphanumerics, underscores, or hyphens. Convert to lowercase. Also strip leading and trailing whitespace. """ @@ -293,7 +294,7 @@ def strip_accents(s: str) -> str: def uri_to_path(uri: str) -> str: """ - Convert a file URI (e.g. 'file:///tmp') to a local path (e.g. '/tmp') + Convert a file URI (e.g. ``file:///tmp``) to a local path (e.g. ``/tmp``) """ if not uri.startswith("file"): raise ValueError("A file URI must be provided (e.g. 'file:///tmp'") @@ -328,10 +329,10 @@ def mutate_dict_in_place(func: Callable[[Any], Any], mapping: Dict[Any, Any]) -> def merge_mappings(mapping1: Dict[Any, Any], mapping2: Dict[Any, Any]) -> None: - """Merge two mappings with string keys, values from `mapping2` overriding values - from `mapping1`. + """Merge two mappings with string keys, values from ``mapping2`` overriding values + from ``mapping1``. - Do its best to detect the key in `mapping1` to override. For example:: + Do its best to detect the key in ``mapping1`` to override. For example: >>> mapping2 = {"keya": "new"} >>> mapping1 = {"keyA": "obsolete"} @@ -339,12 +340,11 @@ def merge_mappings(mapping1: Dict[Any, Any], mapping2: Dict[Any, Any]) -> None: >>> mapping1 {'keyA': 'new'} - If mapping2 has a key that cannot be detected in mapping1, this new key is added - to mapping1 as is. + If ``mapping2`` has a key that cannot be detected in ``mapping1``, this new key is + added to ``mapping1`` as is. :param mapping1: The mapping containing values to be overridden - :param mapping2: The mapping containing values that will override the - first mapping + :param mapping2: The mapping containing values that will override the first mapping """ # A mapping between mapping1 keys as lowercase strings and original mapping1 keys m1_keys_lowercase = {key.lower(): key for key in mapping1} @@ -411,7 +411,7 @@ def get_timestamp(date_time: str) -> float: If the datetime has no offset, it is assumed to be an UTC datetime. :param date_time: The datetime string to return as timestamp - :returns: The timestamp corresponding to the date_time string in seconds + :returns: The timestamp corresponding to the ``date_time`` string in seconds """ dt = isoparse(date_time) if not dt.tzinfo: @@ -420,7 +420,7 @@ def get_timestamp(date_time: str) -> float: def datetime_range(start: dt, end: dt) -> Iterator[dt]: - """Generator function for all dates in-between start and end date.""" + """Generator function for all dates in-between ``start`` and ``end`` date.""" delta = end - start for nday in range(delta.days + 1): yield start + datetime.timedelta(days=nday) @@ -440,15 +440,15 @@ def __call__(self, product: EOProduct) -> None: class ProgressCallback(tqdm): """A callable used to render progress to users for long running processes. - It inherits from `tqdm.auto.tqdm`, and accepts the same arguments on - instantiation: `iterable`, `desc`, `total`, `leave`, `file`, `ncols`, - `mininterval`, `maxinterval`, `miniters`, `ascii`, `disable`, `unit`, - `unit_scale`, `dynamic_ncols`, `smoothing`, `bar_format`, `initial`, - `position`, `postfix`, `unit_divisor`. + It inherits from :class:`tqdm.auto.tqdm`, and accepts the same arguments on + instantiation: ``iterable``, ``desc``, ``total``, ``leave``, ``file``, ``ncols``, + ``mininterval``, ``maxinterval``, ``miniters``, ``ascii``, ``disable``, ``unit``, + ``unit_scale``, ``dynamic_ncols``, ``smoothing``, ``bar_format``, ``initial``, + ``position``, ``postfix``, ``unit_divisor``. - It can be globally disabled using `eodag.utils.logging.setup_logging(0)` or - `eodag.utils.logging.setup_logging(level, no_progress_bar=True)`, and - individually disabled using `disable=True`. + It can be globally disabled using ``eodag.utils.logging.setup_logging(0)`` or + ``eodag.utils.logging.setup_logging(level, no_progress_bar=True)``, and + individually disabled using ``disable=True``. """ def __init__(self, *args: Any, **kwargs: Any) -> None: @@ -483,8 +483,8 @@ def copy(self, *args: Any, **kwargs: Any) -> ProgressCallback: """Returns another progress callback using the same initial keyword-arguments. - Optional `args` and `kwargs` parameters will be used to create a - new `~eodag.utils.ProgressCallback` instance, overriding initial + Optional ``args`` and ``kwargs`` parameters will be used to create a + new :class:`~eodag.utils.ProgressCallback` instance, overriding initial `kwargs`. """ @@ -506,7 +506,7 @@ def get_progress_callback() -> tqdm: def repeatfunc(func: Callable[..., Any], n: int, *args: Any) -> starmap: - """Call `func` `n` times with `args`""" + """Call ``func`` ``n`` times with ``args``""" return starmap(func, repeat(args, n)) @@ -521,12 +521,12 @@ def makedirs(dirpath: str) -> None: def rename_subfolder(dirpath: str, name: str) -> None: - """Rename first subfolder found in dirpath with given name, - raise RuntimeError if no subfolder can be found + """Rename first subfolder found in ``dirpath`` with given ``name``, + raise :class:`RuntimeError` if no subfolder can be found :param dirpath: path to the directory containing the subfolder :param name: new name of the subfolder - :raises: RuntimeError + :raises: :class:`RuntimeError` Example: @@ -540,16 +540,20 @@ def rename_subfolder(dirpath: str, name: str) -> None: ... rename_subfolder(tmpdir, "otherfolder") ... assert not os.path.isdir(somefolder) and os.path.isdir(otherfolder) - Before: + Before:: + $ tree └── somefolder └── somefile - After: + + After:: + $ tree └── otherfolder └── somefile + """ try: subdir, *_ = (p for p in glob(os.path.join(dirpath, "*")) if os.path.isdir(p)) @@ -565,7 +569,7 @@ def rename_subfolder(dirpath: str, name: str) -> None: def format_dict_items( config_dict: Dict[str, Any], **format_variables: Any ) -> Dict[Any, Any]: - r"""Recursive apply string.format(\**format_variables) to dict elements + r"""Recursively apply :meth:`str.format` to ``**format_variables`` on ``config_dict`` values >>> format_dict_items( ... {"foo": {"bar": "{a}"}, "baz": ["{b}?", "{b}!"]}, @@ -583,7 +587,7 @@ def format_dict_items( def jsonpath_parse_dict_items( jsonpath_dict: Dict[str, Any], values_dict: Dict[str, Any] ) -> Dict[Any, Any]: - """Recursive parse jsonpath elements in dict + """Recursively parse :class:`jsonpath_ng.JSONPath` elements in dict >>> import jsonpath_ng.ext as jsonpath >>> jsonpath_parse_dict_items( @@ -592,7 +596,7 @@ def jsonpath_parse_dict_items( ... ) == {'foo': {'bar': 'baz'}, 'qux': ['quux', 'quux']} True - :param jsonpath_dict: Dictionary having values that need to be parsed + :param jsonpath_dict: Dictionary having :class:`jsonpath_ng.JSONPath` values that need to be parsed :param values_dict: Values dict used as args for parsing :returns: Updated dict """ @@ -606,7 +610,7 @@ def update_nested_dict( allow_empty_values: bool = False, allow_extend_duplicates: bool = True, ) -> Dict[Any, Any]: - """Update recursively old_dict items with new_dict ones + """Update recursively ``old_dict`` items with ``new_dict`` ones >>> update_nested_dict( ... {"a": {"a.a": 1, "a.b": 2}, "b": 3}, @@ -868,7 +872,7 @@ def list_items_recursive_sort(config_list: List[Any]) -> List[Any]: def string_to_jsonpath(*args: Any, force: bool = False) -> Union[str, JSONPath]: - """Get jsonpath for "$.foo.bar" like string + """Get :class:`jsonpath_ng.JSONPath` for ``$.foo.bar`` like string >>> string_to_jsonpath(None, "$.foo.bar") Child(Child(Root(), Fields('foo')), Fields('bar')) @@ -882,7 +886,7 @@ def string_to_jsonpath(*args: Any, force: bool = False) -> Union[str, JSONPath]: Fields('foo') :param args: Last arg as input string value, to be converted - :param force: force conversion even if input string is not detected as a jsonpath + :param force: force conversion even if input string is not detected as a :class:`jsonpath_ng.JSONPath` :returns: Parsed value """ path_str: str = args[-1] @@ -945,7 +949,7 @@ def string_to_jsonpath(*args: Any, force: bool = False) -> Union[str, JSONPath]: def format_string(key: str, str_to_format: Any, **format_variables: Any) -> Any: - """Format "{foo}" like string + """Format ``"{foo}"``-like string >>> format_string(None, "foo {bar}, {baz} ?", **{"bar": "qux", "baz": "quux"}) 'foo qux, quux ?' @@ -983,7 +987,7 @@ def format_string(key: str, str_to_format: Any, **format_variables: Any) -> Any: def parse_jsonpath( key: str, jsonpath_obj: Union[str, jsonpath.Child], **values_dict: Dict[str, Any] ) -> Optional[str]: - """Parse jsonpah in jsonpath_obj using values_dict + """Parse jsonpah in ``jsonpath_obj`` using ``values_dict`` >>> import jsonpath_ng.ext as jsonpath >>> parse_jsonpath(None, parse("$.foo.bar"), **{"foo": {"bar": "baz"}}) @@ -1025,10 +1029,10 @@ def nested_pairs2dict(pairs: Union[List[Any], Any]) -> Union[Any, Dict[Any, Any] def get_geometry_from_various( locations_config: List[Dict[str, Any]] = [], **query_args: Any ) -> BaseGeometry: - """Creates a shapely geometry using given query kwargs arguments + """Creates a ``shapely.geometry`` using given query kwargs arguments :param locations_config: (optional) EODAG locations configuration - :param query_args: Query kwargs arguments from core.search() method + :param query_args: Query kwargs arguments from :meth:`~eodag.api.core.EODataAccessGateway.search` :returns: shapely Geometry found :raises: :class:`ValueError` """ @@ -1158,7 +1162,7 @@ def obj_md5sum(data: Any) -> str: @functools.lru_cache() def cached_parse(str_to_parse: str) -> JSONPath: - """Cached jsonpath_ng.ext.parse + """Cached :func:`jsonpath_ng.ext.parse` >>> cached_parse.cache_clear() >>> cached_parse("$.foo") @@ -1174,8 +1178,8 @@ def cached_parse(str_to_parse: str) -> JSONPath: >>> cached_parse.cache_info() CacheInfo(hits=1, misses=2, maxsize=128, currsize=2) - :param str_to_parse: string to parse as jsonpath - :returns: parsed jsonpath + :param str_to_parse: string to parse as :class:`jsonpath_ng.JSONPath` + :returns: parsed :class:`jsonpath_ng.JSONPath` """ return parse(str_to_parse) @@ -1189,7 +1193,7 @@ def _mutable_cached_yaml_load(config_path: str) -> Any: def cached_yaml_load(config_path: str) -> Dict[str, Any]: - """Cached yaml.load + """Cached :func:`yaml.load` :param config_path: path to the yaml configuration file :returns: loaded yaml configuration @@ -1204,7 +1208,7 @@ def _mutable_cached_yaml_load_all(config_path: str) -> List[Any]: def cached_yaml_load_all(config_path: str) -> List[Any]: - """Cached yaml.load_all + """Cached :func:`yaml.load_all` Load all configurations stored in the configuration file as separated yaml documents @@ -1269,7 +1273,8 @@ def flatten_top_directories( def deepcopy(sth: Any) -> Any: """Customized and faster deepcopy inspired by https://stackoverflow.com/a/45858907 - `_copy_list` and `_copy_dict` available for the moment + + ``_copy_list`` and ``_copy_dict`` dispatchers available for the moment :param sth: Object to copy :returns: Copied object @@ -1334,7 +1339,7 @@ def cast_scalar_value(value: Any, new_type: Any) -> Any: :param value: the scalar value to convert :param new_type: the wanted type - :returns: scalar value converted to new_type + :returns: scalar ``value`` converted to ``new_type`` """ if isinstance(value, str) and new_type is bool: # Bool is a type with special meaning in Python, thus the special @@ -1380,8 +1385,9 @@ def guess_extension(type: str) -> Optional[str]: def get_ssl_context(ssl_verify: bool) -> ssl.SSLContext: """ - Returns an SSL context based on ssl_verify argument. - :param ssl_verify: ssl_verify parameter + Returns an SSL context based on ``ssl_verify`` argument. + + :param ssl_verify: :attr:`~eodag.config.PluginConfig.ssl_verify` parameter :returns: An SSL context object. """ ctx = ssl.create_default_context()