From 3f2236e3d6b5304291c6f0ea31d88bf6eaaf536f Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Tue, 10 Dec 2024 09:21:10 +0100 Subject: [PATCH] feat(plugins): order and poll without downloading --- eodag/plugins/apis/usgs.py | 2 +- eodag/plugins/download/base.py | 22 +++++----- eodag/plugins/download/http.py | 61 ++++++++++++++++++++-------- eodag/plugins/download/s3rest.py | 8 ++-- eodag/rest/core.py | 10 ++--- tests/units/test_download_plugins.py | 48 +++++++++------------- 6 files changed, 84 insertions(+), 67 deletions(-) diff --git a/eodag/plugins/apis/usgs.py b/eodag/plugins/apis/usgs.py index 79bf49bdf..f1663e3f6 100644 --- a/eodag/plugins/apis/usgs.py +++ b/eodag/plugins/apis/usgs.py @@ -375,7 +375,7 @@ def download( logger.debug(f"Downloading {req_url}") ssl_verify = getattr(self.config, "ssl_verify", True) - @self._download_retry(product, wait, timeout) + @self._order_download_retry(product, wait, timeout) def download_request( product: EOProduct, fs_path: str, diff --git a/eodag/plugins/download/base.py b/eodag/plugins/download/base.py index b52e0d7a2..12b28355c 100644 --- a/eodag/plugins/download/base.py +++ b/eodag/plugins/download/base.py @@ -589,14 +589,14 @@ def download_all( return paths - def _download_retry( + def _order_download_retry( self, product: EOProduct, wait: int, timeout: int ) -> Callable[[Callable[..., T]], Callable[..., T]]: """ - Download retry decorator. + Order download retry decorator. - Retries the wrapped download method after `wait` minutes if a NotAvailableError - exception is thrown until `timeout` minutes. + Retries the wrapped order_download method after ``wait`` minutes if a + ``NotAvailableError`` exception is thrown until ``timeout`` minutes. :param product: The EO product to download :param wait: If download fails, wait time in minutes between two download tries @@ -605,7 +605,7 @@ def _download_retry( :returns: decorator """ - def decorator(download: Callable[..., T]) -> Callable[..., T]: + def decorator(order_download: Callable[..., T]) -> Callable[..., T]: def download_and_retry(*args: Any, **kwargs: Unpack[DownloadConf]) -> T: # initiate retry loop start_time = datetime.now() @@ -622,7 +622,7 @@ def download_and_retry(*args: Any, **kwargs: Unpack[DownloadConf]) -> T: if datetime_now >= product.next_try: product.next_try += timedelta(minutes=wait) try: - return download(*args, **kwargs) + return order_download(*args, **kwargs) except NotAvailableError as e: if not getattr(self.config, "order_enabled", False): @@ -638,7 +638,7 @@ def download_and_retry(*args: Any, **kwargs: Unpack[DownloadConf]) -> T: ).seconds retry_count += 1 retry_info = ( - f"[Retry #{retry_count}] Waited {wait_seconds}s, trying again to download ordered product" + f"[Retry #{retry_count}] Waited {wait_seconds}s, checking order status again" f" (retry every {wait}' for {timeout}')" ) logger.info(not_available_info) @@ -660,8 +660,8 @@ def download_and_retry(*args: Any, **kwargs: Unpack[DownloadConf]) -> T: ).microseconds / 1e6 retry_count += 1 retry_info = ( - f"[Retry #{retry_count}] Waiting {wait_seconds}s until next download try" - f" for ordered product (retry every {wait}' for {timeout}')" + f"[Retry #{retry_count}] Waiting {wait_seconds}s until next order status check" + f" (retry every {wait}' for {timeout}')" ) logger.info(not_available_info) # Retry-After info from Response header @@ -682,12 +682,12 @@ def download_and_retry(*args: Any, **kwargs: Unpack[DownloadConf]) -> T: logger.info(not_available_info) raise NotAvailableError( f"{product.properties['title']} is not available ({product.properties['storageStatus']})" - f" and could not be downloaded, timeout reached" + f" and order was not successfull, timeout reached" ) elif datetime_now >= stop_time: raise NotAvailableError(not_available_info) - return download(*args, **kwargs) + return order_download(*args, **kwargs) return download_and_retry diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py index 88427cc27..7d3af0943 100644 --- a/eodag/plugins/download/http.py +++ b/eodag/plugins/download/http.py @@ -149,7 +149,7 @@ class HTTPDownload(Download): def __init__(self, provider: str, config: PluginConfig) -> None: super(HTTPDownload, self).__init__(provider, config) - def order_download( + def _order( self, product: EOProduct, auth: Optional[AuthBase] = None, @@ -273,7 +273,7 @@ def order_response_process( return json_response - def order_download_status( + def _order_status( self, product: EOProduct, auth: Optional[AuthBase] = None, @@ -627,7 +627,7 @@ def download( url = product.remote_location - @self._download_retry(product, wait, timeout) + @self._order_download_retry(product, wait, timeout) def download_request( product: EOProduct, auth: AuthBase, @@ -902,6 +902,44 @@ def _process_exception( else: logger.error("Error while getting resource :\n%s", tb.format_exc()) + def _order_request( + self, + product: EOProduct, + auth: Optional[AuthBase], + ) -> None: + if ( + "orderLink" in product.properties + and product.properties.get("storageStatus") == OFFLINE_STATUS + and not product.properties.get("orderStatus") + ): + self._order(product=product, auth=auth) + + if ( + product.properties.get("orderStatusLink", None) + and product.properties.get("storageStatus") != ONLINE_STATUS + ): + self._order_status(product=product, auth=auth) + + def order( + self, + product: EOProduct, + auth: Optional[Union[AuthBase, Dict[str, str]]] = None, + wait: int = DEFAULT_DOWNLOAD_WAIT, + timeout: int = DEFAULT_DOWNLOAD_TIMEOUT, + ) -> None: + """ + Order product and poll to check its status + + :param product: The EO product to download + :param auth: (optional) authenticated object + :param wait: (optional) Wait time in minutes between two order status check + :param timeout: (optional) Maximum time in minutes before stop checking + order status + """ + self._order_download_retry(product, wait, timeout)(self._order_request)( + product, auth + ) + def _stream_download( self, product: EOProduct, @@ -910,8 +948,9 @@ def _stream_download( **kwargs: Unpack[DownloadConf], ) -> Iterator[Any]: """ - fetches a zip file containing the assets of a given product as a stream + Fetches a zip file containing the assets of a given product as a stream and returns a generator yielding the chunks of the file + :param product: product for which the assets should be downloaded :param auth: The configuration of a plugin of type Authentication :param progress_callback: A method or a callable object @@ -928,18 +967,8 @@ def _stream_download( ssl_verify = getattr(self.config, "ssl_verify", True) ordered_message = "" - if ( - "orderLink" in product.properties - and product.properties.get("storageStatus") == OFFLINE_STATUS - and not product.properties.get("orderStatus") - ): - self.order_download(product=product, auth=auth) - - if ( - product.properties.get("orderStatusLink", None) - and product.properties.get("storageStatus") != ONLINE_STATUS - ): - self.order_download_status(product=product, auth=auth) + # retry handled at download level + self._order_request(product, auth) params = kwargs.pop("dl_url_params", None) or getattr( self.config, "dl_url_params", {} diff --git a/eodag/plugins/download/s3rest.py b/eodag/plugins/download/s3rest.py index 1158a70e8..162e9a918 100644 --- a/eodag/plugins/download/s3rest.py +++ b/eodag/plugins/download/s3rest.py @@ -130,9 +130,9 @@ def download( and "storageStatus" in product.properties and product.properties["storageStatus"] != ONLINE_STATUS ): - self.http_download_plugin.order_download(product=product, auth=auth) + self.http_download_plugin._order(product=product, auth=auth) - @self._download_retry(product, wait, timeout) + @self._order_download_retry(product, wait, timeout) def download_request( product: EOProduct, auth: AuthBase, @@ -142,9 +142,7 @@ def download_request( ): # check order status if product.properties.get("orderStatusLink", None): - self.http_download_plugin.order_download_status( - product=product, auth=auth - ) + self.http_download_plugin._order_status(product=product, auth=auth) # get bucket urls bucket_name, prefix = get_bucket_name_and_prefix( diff --git a/eodag/rest/core.py b/eodag/rest/core.py index 82b381816..c8f065f3b 100644 --- a/eodag/rest/core.py +++ b/eodag/rest/core.py @@ -327,13 +327,11 @@ def _order_and_update( if ( product.properties.get("storageStatus") != ONLINE_STATUS and NOT_AVAILABLE in product.properties.get("orderStatusLink", "") - and hasattr(product.downloader, "order_download") + and hasattr(product.downloader, "_order") ): # first order logger.debug("Order product") - order_status_dict = product.downloader.order_download( - product=product, auth=auth - ) + order_status_dict = product.downloader._order(product=product, auth=auth) query_args.update(order_status_dict or {}) if ( @@ -344,11 +342,11 @@ def _order_and_update( product.properties["storageStatus"] = STAGING_STATUS if product.properties.get("storageStatus") == STAGING_STATUS and hasattr( - product.downloader, "order_download_status" + product.downloader, "_order_status" ): # check order status if needed logger.debug("Checking product order status") - product.downloader.order_download_status(product=product, auth=auth) + product.downloader._order_status(product=product, auth=auth) if product.properties.get("storageStatus") != ONLINE_STATUS: raise NotAvailableError("Product is not available yet") diff --git a/tests/units/test_download_plugins.py b/tests/units/test_download_plugins.py index 603b5e42f..4bf19d25a 100644 --- a/tests/units/test_download_plugins.py +++ b/tests/units/test_download_plugins.py @@ -1188,7 +1188,7 @@ def run(): @mock.patch("eodag.plugins.download.http.requests.request", autospec=True) def test_plugins_download_http_order_get(self, mock_request): - """HTTPDownload.order_download() must request using orderLink and GET protocol""" + """HTTPDownload._order() must request using orderLink and GET protocol""" plugin = self.get_download_plugin(self.product) self.product.properties["downloadLink"] = "https://peps.cnes.fr/dummy" self.product.properties["orderLink"] = "http://somewhere/order" @@ -1202,7 +1202,7 @@ def test_plugins_download_http_order_get(self, mock_request): auth_plugin.config.credentials = {"username": "foo", "password": "bar"} auth = auth_plugin.authenticate() - plugin.order_download(self.product, auth=auth) + plugin._order(self.product, auth=auth) mock_request.assert_called_once_with( method="GET", @@ -1220,7 +1220,7 @@ def test_plugins_download_http_order_get(self, mock_request): @mock.patch("eodag.plugins.download.http.requests.request", autospec=True) def test_plugins_download_http_order_get_raises_if_request_500(self, mock_request): - """HTTPDownload.order_download() must raise an error if request to backend + """HTTPDownload._order() must raise an error if request to backend provider failed""" # Configure mock to raise an error @@ -1236,7 +1236,7 @@ def test_plugins_download_http_order_get_raises_if_request_500(self, mock_reques # Verify that a DownloadError is raised with self.assertRaises(DownloadError) as context: - plugin.order_download(self.product, auth=auth) + plugin._order(self.product, auth=auth) self.assertIn("could not be ordered", str(context.exception)) mock_request.assert_called_once_with( @@ -1267,7 +1267,7 @@ def test_plugins_download_http_order_get_raises_if_request_400(self, mock_reques # Test the function, expecting ValidationError to be raised with self.assertRaises(ValidationError) as context: - plugin.order_download(self.product, auth=auth) + plugin._order(self.product, auth=auth) self.assertIn("could not be ordered", str(context.exception)) mock_request.assert_called_once_with( @@ -1281,7 +1281,7 @@ def test_plugins_download_http_order_get_raises_if_request_400(self, mock_reques @mock.patch("eodag.plugins.download.http.requests.request", autospec=True) def test_plugins_download_http_order_post(self, mock_request): - """HTTPDownload.order_download() must request using orderLink and POST protocol""" + """HTTPDownload._order() must request using orderLink and POST protocol""" plugin = self.get_download_plugin(self.product) self.product.properties["downloadLink"] = "https://peps.cnes.fr/dummy" self.product.properties["storageStatus"] = OFFLINE_STATUS @@ -1293,7 +1293,7 @@ def test_plugins_download_http_order_post(self, mock_request): # orderLink without query query args self.product.properties["orderLink"] = "http://somewhere/order" - plugin.order_download(self.product, auth=auth) + plugin._order(self.product, auth=auth) mock_request.assert_called_once_with( method="POST", url=self.product.properties["orderLink"], @@ -1305,7 +1305,7 @@ def test_plugins_download_http_order_post(self, mock_request): # orderLink with query query args mock_request.reset_mock() self.product.properties["orderLink"] = "http://somewhere/order?foo=bar" - plugin.order_download(self.product, auth=auth) + plugin._order(self.product, auth=auth) mock_request.assert_called_once_with( method="POST", url="http://somewhere/order", @@ -1321,7 +1321,7 @@ def test_plugins_download_http_order_post(self, mock_request): self.product.properties[ "orderLink" ] = 'http://somewhere/order?{"location": "dataset_id=lorem&data_version=202211", "cacheable": "true"}' - plugin.order_download(self.product, auth=auth) + plugin._order(self.product, auth=auth) mock_request.assert_called_once_with( method="POST", url="http://somewhere/order", @@ -1336,7 +1336,7 @@ def test_plugins_download_http_order_post(self, mock_request): ) def test_plugins_download_http_order_status(self): - """HTTPDownload.order_download_status() must request status using orderStatusLink""" + """HTTPDownload._order_status() must request status using orderStatusLink""" plugin = self.get_download_plugin(self.product) plugin.config.order_status = { "metadata_mapping": { @@ -1363,7 +1363,7 @@ def run(): ) with self.assertRaises(DownloadError): - plugin.order_download_status(self.product, auth=auth) + plugin._order_status(self.product, auth=auth) self.assertIn( list(USER_AGENT.items())[0], responses.calls[0].request.headers.items() @@ -1376,7 +1376,7 @@ def run(): def test_plugins_download_http_order_status_get_raises_if_request_500( self, mock_request ): - """HTTPDownload.order_download() must raise an error if request to backend + """HTTPDownload._order() must raise an error if request to backend provider failed""" # Configure mock to raise an error @@ -1393,7 +1393,7 @@ def test_plugins_download_http_order_status_get_raises_if_request_500( # Verify that a DownloadError is raised with self.assertRaises(DownloadError) as context: - plugin.order_download_status(self.product, auth=auth) + plugin._order_status(self.product, auth=auth) self.assertIn("order status could not be checked", str(context.exception)) mock_request.assert_called_once_with( @@ -1428,7 +1428,7 @@ def test_plugins_download_http_order_status_get_raises_if_request_400( # Test the function, expecting ValidationError to be raised with self.assertRaises(ValidationError) as context: - plugin.order_download_status(self.product, auth=auth) + plugin._order_status(self.product, auth=auth) self.assertIn("order status could not be checked", str(context.exception)) mock_request.assert_called_once_with( @@ -1442,7 +1442,7 @@ def test_plugins_download_http_order_status_get_raises_if_request_400( ) def test_plugins_download_http_order_status_search_again(self): - """HTTPDownload.order_download_status() must search again after success if needed""" + """HTTPDownload._order_status() must search again after success if needed""" plugin = self.get_download_plugin(self.product) plugin.config.order_status = { "metadata_mapping": {"status": "$.json.status"}, @@ -1484,7 +1484,7 @@ def run(): ), ) - plugin.order_download_status(self.product, auth=auth) + plugin._order_status(self.product, auth=auth) self.assertEqual( self.product.properties["downloadLink"], "http://new-download-link" @@ -2020,12 +2020,8 @@ def setUp(self): productType="S2_MSI_L1C", ) - @mock.patch( - "eodag.plugins.download.http.HTTPDownload.order_download_status", autospec=True - ) - @mock.patch( - "eodag.plugins.download.http.HTTPDownload.order_download", autospec=True - ) + @mock.patch("eodag.plugins.download.http.HTTPDownload._order_status", autospec=True) + @mock.patch("eodag.plugins.download.http.HTTPDownload._order", autospec=True) def test_plugins_download_s3rest_online(self, mock_order, mock_order_status): """S3RestDownload.download() must create outputfiles""" @@ -2092,12 +2088,8 @@ def run(): mock_order.assert_not_called() mock_order_status.assert_not_called() - @mock.patch( - "eodag.plugins.download.http.HTTPDownload.order_download_status", autospec=True - ) - @mock.patch( - "eodag.plugins.download.http.HTTPDownload.order_download", autospec=True - ) + @mock.patch("eodag.plugins.download.http.HTTPDownload._order_status", autospec=True) + @mock.patch("eodag.plugins.download.http.HTTPDownload._order", autospec=True) def test_plugins_download_s3rest_offline(self, mock_order, mock_order_status): """S3RestDownload.download() must order offline products"""