diff --git a/.gitignore b/.gitignore index 0fcb9b900..c01c2553f 100644 --- a/.gitignore +++ b/.gitignore @@ -94,6 +94,7 @@ ENV/ # Autogenerated docs by sphinx docs/plugins_reference/generated/*.rst +docs/drivers_generated/*.rst docs/notebooks/api_user_guide/*workspace* docs/notebooks/tutos/*workspace* diff --git a/docs/api_reference/eoproduct.rst b/docs/api_reference/eoproduct.rst index 96325f0ec..fe0174db4 100644 --- a/docs/api_reference/eoproduct.rst +++ b/docs/api_reference/eoproduct.rst @@ -19,6 +19,13 @@ Download EOProduct.download EOProduct.get_quicklook +Driver +-------- + +.. autosummary:: + + EOProduct.driver + Conversion ---------- @@ -36,4 +43,4 @@ Interface .. autoclass:: eodag.api.product._product.EOProduct - :members: download, get_quicklook, as_dict, from_geojson, __geo_interface__ + :members: driver, download, get_quicklook, as_dict, from_geojson, __geo_interface__ diff --git a/docs/conf.py b/docs/conf.py index dd980df07..c8e6320cb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -281,12 +281,19 @@ def _shorten_titles(dir_path): r"\1", file_content, ) + # remove long prefix from drivers titles + file_content = re.sub( + r"(\.html\">|\"#\">|

)eodag\.api\.product\.drivers\.[a-z0-9]+\.", + r"\1", + file_content, + ) # write file.seek(0) file.write(file_content) - print(f"Plugins titles shortened in {file_path}") + print(f"Titles shortened in {file_path}") _shorten_titles(os.path.join(app.outdir, "plugins_reference")) + _shorten_titles(os.path.join(app.outdir, "drivers_generated")) def _html_page_context(app, pagename, templatename, context, doctree): diff --git a/docs/drivers.rst b/docs/drivers.rst new file mode 100644 index 000000000..1dc38a629 --- /dev/null +++ b/docs/drivers.rst @@ -0,0 +1,46 @@ +.. module:: eodag.api.product.drivers + +Drivers +======= + +Drivers enable additional methods to be called on the :class:`~eodag.api.product._product.EOProduct`. They are set as +:attr:`~eodag.api.product._product.EOProduct.driver` attribute of the :class:`~eodag.api.product._product.EOProduct` +during its initialization, using some criteria to determine the most adapted driver. The first driver having its +associated criteria matching will be used. If no driver is found, the :class:`~eodag.api.product.drivers.base.NoDriver` +criteria is used. + + +Criteria +^^^^^^^^ + +.. autoclass:: eodag.api.product.drivers.DriverCriteria + :members: + +.. autodata:: DRIVERS + :no-value: +.. autodata:: LEGACY_DRIVERS + :no-value: + + +Methods available +^^^^^^^^^^^^^^^^^ + +.. autoclass:: eodag.api.product.drivers.base.DatasetDriver + :members: + :member-order: bysource + +.. autoclass:: eodag.api.product.drivers.base.AssetPatterns + :members: + +Drivers Available +^^^^^^^^^^^^^^^^^ + +EODAG currently advertises the following drivers: + +.. autosummary:: + :toctree: drivers_generated/ + + base.NoDriver + generic.GenericDriver + sentinel1.Sentinel1Driver + sentinel2.Sentinel2Driver diff --git a/docs/index.rst b/docs/index.rst index e961949fe..6d7d5cd30 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -132,6 +132,7 @@ Site contents add_provider add_product_type plugins + drivers params_mapping contribute changelog diff --git a/eodag/api/product/_product.py b/eodag/api/product/_product.py index a6577d330..4cc2030a5 100644 --- a/eodag/api/product/_product.py +++ b/eodag/api/product/_product.py @@ -38,7 +38,7 @@ except ImportError: from eodag.api.product._assets import AssetsDict -from eodag.api.product.drivers import DRIVERS, NoDriver +from eodag.api.product.drivers import DRIVERS, LEGACY_DRIVERS, NoDriver from eodag.api.product.metadata_mapping import ( DEFAULT_GEOMETRY, NOT_AVAILABLE, @@ -122,6 +122,8 @@ class EOProduct: geometry: BaseGeometry search_intersection: Optional[BaseGeometry] assets: AssetsDict + #: Driver enables additional methods to be called on the EOProduct + driver: DatasetDriver def __init__( self, provider: str, properties: dict[str, Any], **kwargs: Any @@ -498,11 +500,16 @@ def get_driver(self) -> DatasetDriver: try: for driver_conf in DRIVERS: if all([criteria(self) for criteria in driver_conf["criteria"]]): - return driver_conf["driver"] + driver = driver_conf["driver"] + break + # use legacy driver for deprecated get_data method usage + for lecacy_conf in LEGACY_DRIVERS: + if all([criteria(self) for criteria in lecacy_conf["criteria"]]): + driver.legacy = lecacy_conf["driver"] + break + return driver except TypeError: - logger.warning( - "Drivers definition seems out-of-date, please update eodag-cube" - ) + logger.info("No driver matching") pass return NoDriver() diff --git a/eodag/api/product/drivers/__init__.py b/eodag/api/product/drivers/__init__.py index bb1b14d78..7f8151ca1 100644 --- a/eodag/api/product/drivers/__init__.py +++ b/eodag/api/product/drivers/__init__.py @@ -16,14 +16,91 @@ # See the License for the specific language governing permissions and # limitations under the License. """EODAG drivers package""" +from __future__ import annotations + +from typing import Callable, TypedDict + from eodag.api.product.drivers.base import DatasetDriver, NoDriver +from eodag.api.product.drivers.generic import GenericDriver +from eodag.api.product.drivers.sentinel1 import Sentinel1Driver +from eodag.api.product.drivers.sentinel2 import Sentinel2Driver try: - from eodag_cube.api.product.drivers import ( # pyright: ignore[reportMissingImports] - DRIVERS, + # import from eodag-cube if installed + from eodag_cube.api.product.drivers.generic import ( # pyright: ignore[reportMissingImports]; isort: skip + GenericDriver as GenericDriver_cube, + ) + from eodag_cube.api.product.drivers.sentinel2_l1c import ( # pyright: ignore[reportMissingImports]; isort: skip + Sentinel2L1C as Sentinel2L1C_cube, + ) + from eodag_cube.api.product.drivers.stac_assets import ( # pyright: ignore[reportMissingImports]; isort: skip + StacAssets as StacAssets_cube, ) except ImportError: - DRIVERS = [] + GenericDriver_cube = NoDriver + Sentinel2L1C_cube = NoDriver + StacAssets_cube = NoDriver + + +class DriverCriteria(TypedDict): + """Driver criteria definition""" + + #: Function that returns True if the driver is suitable for the given :class:`~eodag.api.product._product.EOProduct` + criteria: list[Callable[..., bool]] + #: driver to use + driver: DatasetDriver + + +#: list of drivers and their criteria +DRIVERS: list[DriverCriteria] = [ + { + "criteria": [ + lambda prod: True + if (prod.product_type or "").startswith("S2_MSI_") + else False + ], + "driver": Sentinel2Driver(), + }, + { + "criteria": [ + lambda prod: True + if (prod.product_type or "").startswith("S1_SAR_") + else False + ], + "driver": Sentinel1Driver(), + }, + { + "criteria": [lambda prod: True], + "driver": GenericDriver(), + }, +] + + +#: list of legacy drivers and their criteria +LEGACY_DRIVERS: list[DriverCriteria] = [ + { + "criteria": [ + lambda prod: True if len(getattr(prod, "assets", {})) > 0 else False + ], + "driver": StacAssets_cube(), + }, + { + "criteria": [lambda prod: True if "assets" in prod.properties else False], + "driver": StacAssets_cube(), + }, + { + "criteria": [ + lambda prod: True + if getattr(prod, "product_type") == "S2_MSI_L1C" + else False + ], + "driver": Sentinel2L1C_cube(), + }, + { + "criteria": [lambda prod: True], + "driver": GenericDriver_cube(), + }, +] # exportable content -__all__ = ["DRIVERS", "DatasetDriver", "NoDriver"] +__all__ = ["DRIVERS", "DatasetDriver", "GenericDriver", "NoDriver", "Sentinel2Driver"] diff --git a/eodag/api/product/drivers/base.py b/eodag/api/product/drivers/base.py index 1dd46f46f..5be2e46b9 100644 --- a/eodag/api/product/drivers/base.py +++ b/eodag/api/product/drivers/base.py @@ -17,15 +17,72 @@ # limitations under the License. from __future__ import annotations -from typing import TYPE_CHECKING +import logging +import re +from typing import TYPE_CHECKING, Optional, TypedDict + +from eodag.utils import _deprecated if TYPE_CHECKING: from eodag.api.product import EOProduct +class AssetPatterns(TypedDict): + """Asset patterns definition""" + + #: pattern to match and extract asset key + pattern: re.Pattern + #: roles associated to the asset key + roles: list[str] + + +logger = logging.getLogger("eodag.driver.base") + + class DatasetDriver(metaclass=type): - """Dataset driver""" + """Parent class for all dataset drivers. + + Drivers will provide methods adapted to a given :class:`~eodag.api.product._product.EOProduct` related to predefined + criteria. + """ + + #: legacy driver for deprecated :meth:`~eodag_cube.api.product._product.EOProduct.get_data` method usage + legacy: DatasetDriver + #: list of patterns to match asset keys and roles + ASSET_KEYS_PATTERNS_ROLES: list[AssetPatterns] = [] + + #: strip non-alphanumeric characters at the beginning and end of the key + STRIP_SPECIAL_PATTERN = re.compile(r"^[^A-Z0-9]+|[^A-Z0-9]+$", re.IGNORECASE) + + def _normalize_key(self, key, eo_product): + # default cleanup + norm_key = key.replace(eo_product.properties.get("id", ""), "") + norm_key = re.sub(self.STRIP_SPECIAL_PATTERN, "", norm_key) + + return norm_key + + def guess_asset_key_and_roles( + self, href: str, eo_product: EOProduct + ) -> tuple[Optional[str], Optional[list[str]]]: + """Guess the asset key and roles from the given href. + + :param href: The asset href + :param eo_product: The product to which the asset belongs + :returns: The asset key and roles + """ + for pattern_dict in self.ASSET_KEYS_PATTERNS_ROLES: + if matched := pattern_dict["pattern"].match(href): + extracted_key, roles = ( + "".join([m for m in matched.groups() if m is not None]), + pattern_dict.get("roles"), + ) + normalized_key = self._normalize_key(extracted_key, eo_product) + return normalized_key or extracted_key, roles + logger.debug(f"No key & roles could be guessed for {href}") + return None, None + + @_deprecated(reason="Method used by deprecated get_data", version="3.1.0") def get_data_address(self, eo_product: EOProduct, band: str) -> str: """Retrieve the address of the dataset represented by `eo_product`. @@ -34,12 +91,16 @@ def get_data_address(self, eo_product: EOProduct, band: str) -> str: :returns: An address for the dataset :raises: :class:`~eodag.utils.exceptions.AddressNotFound` :raises: :class:`~eodag.utils.exceptions.UnsupportedDatasetAddressScheme` + + .. deprecated:: 3.1.0 + Method used by deprecated :meth:`~eodag_cube.api.product._product.EOProduct.get_data` """ raise NotImplementedError class NoDriver(DatasetDriver): - """A default driver that does not implement any of the methods it should implement, used for all product types for - which the :meth:`~eodag.api.product._product.EOProduct.get_data` method is not yet implemented in eodag. Expect a + """A default :attr:`~eodag.api.product.drivers.base.DatasetDriver.legacy` driver that does not implement any of the + methods it should implement, used for all product types for which the deprecated + :meth:`~eodag_cube.api.product._product.EOProduct.get_data` method is not implemented. Expect a :exc:`NotImplementedError` when trying to get the data in that case. """ diff --git a/eodag/api/product/drivers/generic.py b/eodag/api/product/drivers/generic.py new file mode 100644 index 000000000..cb8d34224 --- /dev/null +++ b/eodag/api/product/drivers/generic.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +# Copyright 2021, CS GROUP - France, http://www.c-s.fr +# +# This file is part of EODAG project +# https://www.github.com/CS-SI/EODAG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import logging +import re + +from eodag.api.product.drivers.base import AssetPatterns, DatasetDriver + +logger = logging.getLogger("eodag.driver.generic") + + +class GenericDriver(DatasetDriver): + """Generic default Driver""" + + #: list of patterns to match asset keys and roles + ASSET_KEYS_PATTERNS_ROLES: list[AssetPatterns] = [ + # data + { + "pattern": re.compile( + r"^(?:.*[/\\])?([^/\\]+)(\.jp2|\.tiff?|\.dat|\.nc|\.grib2?)$", + re.IGNORECASE, + ), + "roles": ["data"], + }, + # metadata + { + "pattern": re.compile( + r"^(?:.*[/\\])?([^/\\]+)(\.xml|\.xsd|\.safe|\.json)$", re.IGNORECASE + ), + "roles": ["metadata"], + }, + # thumbnail + { + "pattern": re.compile( + r"^(?:.*[/\\])?(thumbnail)(\.jpg|\.jpeg|\.png)$", re.IGNORECASE + ), + "roles": ["thumbnail"], + }, + # quicklook + { + "pattern": re.compile( + r"^(?:.*[/\\])?([^/\\]+-ql|preview|quick-?look)(\.jpg|\.jpeg|\.png)$", + re.IGNORECASE, + ), + "roles": ["overview"], + }, + # default + {"pattern": re.compile(r"^(?:.*[/\\])?([^/\\]+)$"), "roles": ["auxiliary"]}, + ] diff --git a/eodag/api/product/drivers/sentinel1.py b/eodag/api/product/drivers/sentinel1.py new file mode 100644 index 000000000..dc17d18f8 --- /dev/null +++ b/eodag/api/product/drivers/sentinel1.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +# Copyright 2025, CS GROUP - France, http://www.c-s.fr +# +# This file is part of EODAG project +# https://www.github.com/CS-SI/EODAG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +from eodag.api.product.drivers.base import AssetPatterns, DatasetDriver + +if TYPE_CHECKING: + from eodag.api.product._product import EOProduct + + +class Sentinel1Driver(DatasetDriver): + """Driver for Sentinel1 products""" + + #: pattern to match data-role keys + DATA_PATTERN = re.compile(r"[vh]{2}", re.IGNORECASE) + + #: list of patterns to replace in asset keys + REPLACE_PATTERNS = [ + (re.compile(r"s1a?", re.IGNORECASE), ""), + (re.compile(r"grd", re.IGNORECASE), ""), + (re.compile(r"slc", re.IGNORECASE), ""), + (re.compile(r"ocn", re.IGNORECASE), ""), + (re.compile(r"iw", re.IGNORECASE), ""), + (re.compile(r"ew", re.IGNORECASE), ""), + (re.compile(r"wv", re.IGNORECASE), ""), + (re.compile(r"sm", re.IGNORECASE), ""), + (re.compile(r"raw([-_]s)?", re.IGNORECASE), ""), + (re.compile(r"[t?0-9]{3,}", re.IGNORECASE), ""), + (re.compile(r"-+"), "-"), + (re.compile(r"-+\."), "."), + (re.compile(r"_+"), "_"), + (re.compile(r"_+\."), "."), + ] + + #: list of patterns to match asset keys and roles + ASSET_KEYS_PATTERNS_ROLES: list[AssetPatterns] = [ + # data + { + "pattern": re.compile( + r"^.*?([vh]{2}).*\.(?:jp2|tiff?|dat)$", re.IGNORECASE + ), + "roles": ["data"], + }, + # metadata + { + "pattern": re.compile( + r"^(?:.*[/\\])?([^/\\]+)(\.xml|\.xsd|\.safe|\.json)$", re.IGNORECASE + ), + "roles": ["metadata"], + }, + # thumbnail + { + "pattern": re.compile( + r"^(?:.*[/\\])?(thumbnail)(\.jpe?g|\.png)$", re.IGNORECASE + ), + "roles": ["thumbnail"], + }, + # quicklook + { + "pattern": re.compile( + r"^(?:.*[/\\])?([^/\\]+-ql|preview|quick-?look)(\.jpe?g|\.png)$", + re.IGNORECASE, + ), + "roles": ["overview"], + }, + # default + {"pattern": re.compile(r"^(?:.*[/\\])?([^/\\]+)$"), "roles": ["auxiliary"]}, + ] + + def _normalize_key(self, key: str, eo_product: EOProduct) -> str: + if self.DATA_PATTERN.fullmatch(key): + return key.upper() + + key = super()._normalize_key(key, eo_product) + + for pattern, replacement in self.REPLACE_PATTERNS: + key = pattern.sub(replacement, key) + + return super()._normalize_key(key, eo_product) diff --git a/eodag/api/product/drivers/sentinel2.py b/eodag/api/product/drivers/sentinel2.py new file mode 100644 index 000000000..76e617067 --- /dev/null +++ b/eodag/api/product/drivers/sentinel2.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# Copyright 2021, CS GROUP - France, http://www.c-s.fr +# +# This file is part of EODAG project +# https://www.github.com/CS-SI/EODAG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +from eodag.api.product.drivers.base import AssetPatterns, DatasetDriver + +if TYPE_CHECKING: + from eodag.api.product._product import EOProduct + + +class Sentinel2Driver(DatasetDriver): + """Driver for Sentinel2 products""" + + #: Band keys associated with their default Ground Sampling Distance (GSD) + BANDS_DEFAULT_GSD = { + "10M": ("B02", "B03", "B04", "B08", "TCI"), + "20M": ("B05", "B06", "B07", "B11", "B12", "B8A"), + "60M": ("B01", "B09", "B10"), + } + + #: list of patterns to match asset keys and roles + ASSET_KEYS_PATTERNS_ROLES: list[AssetPatterns] = [ + # masks + { + "pattern": re.compile(r"^.*?(MSK_[^/\\]+)\.(?:jp2|tiff?)$", re.IGNORECASE), + "roles": ["data-mask"], + }, + # visual + { + "pattern": re.compile( + r"^.*?(TCI)(_[0-9]+m)?\.(?:jp2|tiff?)$", re.IGNORECASE + ), + "roles": ["visual"], + }, + # bands + { + "pattern": re.compile( + r"^.*?([A-Z]+[0-9]*[A-Z]?)(_[0-9]+m)?\.(?:jp2|tiff?)$", re.IGNORECASE + ), + "roles": ["data"], + }, + # metadata + { + "pattern": re.compile( + r"^(?:.*[/\\])?([^/\\]+)(\.xml|\.xsd|\.safe|\.json)$", re.IGNORECASE + ), + "roles": ["metadata"], + }, + # thumbnail + { + "pattern": re.compile( + r"^(?:.*[/\\])?(thumbnail)(\.jpe?g|\.png)$", re.IGNORECASE + ), + "roles": ["thumbnail"], + }, + # quicklook + { + "pattern": re.compile( + r"^(?:.*[/\\])?[^/\\]+(-ql|preview|quick-?look)(\.jpe?g|\.png)$", + re.IGNORECASE, + ), + "roles": ["overview"], + }, + # default + {"pattern": re.compile(r"^(?:.*[/\\])?([^/\\]+)$"), "roles": ["auxiliary"]}, + ] + + def _normalize_key(self, key: str, eo_product: EOProduct) -> str: + upper_key = key.upper() + # check if key matched any normalized + for res in self.BANDS_DEFAULT_GSD: + if res in upper_key: + for norm_key in self.BANDS_DEFAULT_GSD[res]: + if norm_key in upper_key: + return norm_key + + return super()._normalize_key(key, eo_product) diff --git a/eodag/plugins/search/creodias_s3.py b/eodag/plugins/search/creodias_s3.py index 656a9d766..d5223be13 100644 --- a/eodag/plugins/search/creodias_s3.py +++ b/eodag/plugins/search/creodias_s3.py @@ -16,8 +16,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import os from types import MethodType from typing import Any +from urllib.parse import urlparse import boto3 import botocore @@ -82,32 +84,28 @@ def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth): ) logger.debug("Listing assets in %s", prefix) product.assets = AssetsDict(product) - s3_res = auth.s3_client.list_objects( + s3_objects = auth.s3_client.list_objects( Bucket=config.s3_bucket, Prefix=prefix, MaxKeys=300 ) # check if product path has assets or is already a file - if "Contents" in s3_res: - for asset in s3_res["Contents"]: - asset_basename = ( - asset["Key"].split("/")[-1] - if "/" in asset["Key"] - else asset["Key"] + if "Contents" in s3_objects: + for s3_obj in s3_objects["Contents"]: + key, roles = product.driver.guess_asset_key_and_roles( + s3_obj["Key"], product ) - - if len(asset_basename) > 0 and asset_basename not in product.assets: - role = ( - "data" - if asset_basename.split(".")[-1] in DATA_EXTENSIONS - else "metadata" - ) - - product.assets[asset_basename] = { - "title": asset_basename, - "roles": [role], - "href": f"s3://{config.s3_bucket}/{asset['Key']}", + parsed_url = urlparse(s3_obj["Key"]) + title = os.path.basename(parsed_url.path) + + if key and key not in product.assets: + product.assets[key] = { + "title": title, + "roles": roles, + "href": f"s3://{config.s3_bucket}/{s3_obj['Key']}", } - if mime_type := guess_file_type(asset["Key"]): - product.assets[asset_basename]["type"] = mime_type + if mime_type := guess_file_type(s3_obj["Key"]): + product.assets[key]["type"] = mime_type + # sort assets + product.assets.data = dict(sorted(product.assets.data.items())) # update driver product.driver = product.get_driver() diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index 29e605b9a..2a7dd0547 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -1087,8 +1087,15 @@ def normalize_results( product.properties = dict( getattr(self.config, "product_type_config", {}), **product.properties ) - # move assets from properties to product's attr - product.assets.update(product.properties.pop("assets", {})) + # move assets from properties to product's attr, normalize keys & roles + for key, asset in product.properties.pop("assets", {}).items(): + norm_key, asset["roles"] = product.driver.guess_asset_key_and_roles( + asset.get("href", ""), product + ) + if norm_key: + product.assets[norm_key] = asset + # sort assets + product.assets.data = dict(sorted(product.assets.data.items())) products.append(product) return products diff --git a/tests/context.py b/tests/context.py index 0b64d07fa..2fbb7ceda 100644 --- a/tests/context.py +++ b/tests/context.py @@ -28,7 +28,10 @@ from eodag.api.core import DEFAULT_ITEMS_PER_PAGE, DEFAULT_MAX_ITEMS_PER_PAGE from eodag.api.product import EOProduct from eodag.api.product.drivers import DRIVERS -from eodag.api.product.drivers.base import DatasetDriver +from eodag.api.product.drivers.generic import GenericDriver +from eodag.api.product.drivers.sentinel1 import Sentinel1Driver +from eodag.api.product.drivers.sentinel2 import Sentinel2Driver +from eodag.api.product.drivers.base import DatasetDriver, NoDriver from eodag.api.product.metadata_mapping import ( format_metadata, OFFLINE_STATUS, diff --git a/tests/units/test_eoproduct_driver_generic.py b/tests/units/test_eoproduct_driver_generic.py new file mode 100644 index 000000000..d7e882ae7 --- /dev/null +++ b/tests/units/test_eoproduct_driver_generic.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +# Copyright 2021, CS GROUP - France, http://www.c-s.fr +# +# This file is part of EODAG project +# https://www.github.com/CS-SI/EODAG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from tests import TEST_RESOURCES_PATH, EODagTestCase +from tests.context import EOProduct, GenericDriver, NoDriver + + +class TestEOProductDriverGeneric(EODagTestCase): + def setUp(self): + super(TestEOProductDriverGeneric, self).setUp() + self.product = EOProduct( + self.provider, self.eoproduct_props, productType="FAKE_PRODUCT_TYPE" + ) + self.product.properties["title"] = os.path.join( + TEST_RESOURCES_PATH, + "products", + "S2A_MSIL1C_20180101T105441_N0206_R051_T31TDH_20180101T124911", + ) + + def test_driver_generic_init(self): + """The appropriate driver must have been set""" + self.assertIsInstance(self.product.driver, GenericDriver) + self.assertTrue(hasattr(self.product.driver, "legacy")) + try: + # import from eodag-cube if installed + from eodag_cube.api.product.drivers.base import ( # pyright: ignore[reportMissingImports]; isort: skip + DatasetDriver as DatasetDriver_cube, + ) + + self.assertIsInstance(self.product.driver.legacy, DatasetDriver_cube) + except ImportError: + self.assertIsInstance(self.product.driver.legacy, NoDriver) + + def test_driver_generic_guess_asset_key_and_roles(self): + """The driver must guess appropriate asset key and roles""" + self.assertEqual( + self.product.driver.guess_asset_key_and_roles("", self.product), + (None, None), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "2018/1/28/0/ew-hv.tif", self.product + ), + ("ew-hv.tif", ["data"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "2018/1/28/0/ew-hv.jp2", self.product + ), + ("ew-hv.jp2", ["data"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "2018/1/28/0/ew-hv.nc", self.product + ), + ("ew-hv.nc", ["data"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "2018/1/28/0/ew-hv.grib2", self.product + ), + ("ew-hv.grib2", ["data"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "2018/1/28/0/ew-hv.foo", self.product + ), + ("ew-hv.foo", ["auxiliary"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "s3://foo/1/28/0/rfi-ew-hh.xml", self.product + ), + ("rfi-ew-hh.xml", ["metadata"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "s3://foo/1/28/0/thumbnail.png", self.product + ), + ("thumbnail.png", ["thumbnail"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "s3://foo/1/28/0/quick-look.jpg", self.product + ), + ("quick-look.jpg", ["overview"]), + ) diff --git a/tests/units/test_eoproduct_driver_sentinel1.py b/tests/units/test_eoproduct_driver_sentinel1.py new file mode 100644 index 000000000..a2d30793a --- /dev/null +++ b/tests/units/test_eoproduct_driver_sentinel1.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +# Copyright 2021, CS GROUP - France, http://www.c-s.fr +# +# This file is part of EODAG project +# https://www.github.com/CS-SI/EODAG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from tests import TEST_RESOURCES_PATH, EODagTestCase +from tests.context import EOProduct, NoDriver, Sentinel1Driver + + +class TestEOProductDriverSentinel1Driver(EODagTestCase): + def setUp(self): + super(TestEOProductDriverSentinel1Driver, self).setUp() + self.product = EOProduct( + self.provider, self.eoproduct_props, productType="S1_SAR_OCN" + ) + self.product.properties["title"] = os.path.join( + TEST_RESOURCES_PATH, + "products", + "S2A_MSIL1C_20180101T105441_N0206_R051_T31TDH_20180101T124911", + ) + + def test_driver_s1_init(self): + """The appropriate driver must have been set""" + self.assertIsInstance(self.product.driver, Sentinel1Driver) + self.assertTrue(hasattr(self.product.driver, "legacy")) + try: + # import from eodag-cube if installed + from eodag_cube.api.product.drivers.base import ( # pyright: ignore[reportMissingImports]; isort: skip + DatasetDriver as DatasetDriver_cube, + ) + + self.assertIsInstance(self.product.driver.legacy, DatasetDriver_cube) + except ImportError: + self.assertIsInstance(self.product.driver.legacy, NoDriver) + + def test_driver_s1_guess_asset_key_and_roles(self): + """The driver must guess appropriate asset key and roles""" + self.assertEqual( + self.product.driver.guess_asset_key_and_roles("", self.product), + (None, None), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "2018/1/28/0/ew-hv.tiff", self.product + ), + ("HV", ["data"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "http://foo/1/28/0/iw-hh.tif", self.product + ), + ("HH", ["data"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "s3://foo/1/28/0/s1a-vv.tiff", self.product + ), + ("VV", ["data"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "s3://foo/1/28/0/rfi-ew-hh.xml", self.product + ), + ("rfi-hh.xml", ["metadata"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "s3://foo/1/28/0/thumbnail.png", self.product + ), + ("thumbnail.png", ["thumbnail"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "s3://foo/1/28/0/quick-look.jpg", self.product + ), + ("quick-look.jpg", ["overview"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "s3://foo/1/28/0/foo.bar", self.product + ), + ("foo.bar", ["auxiliary"]), + ) diff --git a/tests/units/test_eoproduct_driver_sentinel2.py b/tests/units/test_eoproduct_driver_sentinel2.py new file mode 100644 index 000000000..3abc37e11 --- /dev/null +++ b/tests/units/test_eoproduct_driver_sentinel2.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# Copyright 2021, CS GROUP - France, http://www.c-s.fr +# +# This file is part of EODAG project +# https://www.github.com/CS-SI/EODAG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from tests import TEST_RESOURCES_PATH, EODagTestCase +from tests.context import EOProduct, NoDriver, Sentinel2Driver + + +class TestEOProductDriverSentinel2Driver(EODagTestCase): + def setUp(self): + super(TestEOProductDriverSentinel2Driver, self).setUp() + self.product = EOProduct( + self.provider, self.eoproduct_props, productType=self.product_type + ) + self.product.properties["title"] = os.path.join( + TEST_RESOURCES_PATH, + "products", + "S2A_MSIL1C_20180101T105441_N0206_R051_T31TDH_20180101T124911", + ) + + def test_driver_s2_init(self): + """The appropriate driver must have been set""" + self.assertIsInstance(self.product.driver, Sentinel2Driver) + self.assertTrue(hasattr(self.product.driver, "legacy")) + try: + # import from eodag-cube if installed + from eodag_cube.api.product.drivers.base import ( # pyright: ignore[reportMissingImports]; isort: skip + DatasetDriver as DatasetDriver_cube, + ) + + self.assertIsInstance(self.product.driver.legacy, DatasetDriver_cube) + except ImportError: + self.assertIsInstance(self.product.driver.legacy, NoDriver) + + def test_driver_s2_guess_asset_key_and_roles(self): + """The driver must guess appropriate asset key and roles""" + self.assertEqual( + self.product.driver.guess_asset_key_and_roles("", self.product), + (None, None), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "tiles/31/T/DJ/2018/1/28/0/bla_TCI.jp2", self.product + ), + ("TCI", ["visual"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "tiles/31/T/DJ/2018/1/28/0/B01.jp2", self.product + ), + ("B01", ["data"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "http://foo/1/28/0/bla_bla-B02_10m.tif", self.product + ), + ("B02", ["data"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "s3://foo/1/28/0/bla_bla-B02_20m.tiff", self.product + ), + ("B02_20m", ["data"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "s3://foo/1/28/0/MSK_bla_B02.tiff", self.product + ), + ("MSK_bla_B02", ["data-mask"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "s3://foo/1/28/0/Foo-bar.xml", self.product + ), + ("Foo-bar.xml", ["metadata"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "s3://foo/1/28/0/thumbnail.png", self.product + ), + ("thumbnail.png", ["thumbnail"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "s3://foo/1/28/0/Foo_bar-ql.jpg", self.product + ), + ("ql.jpg", ["overview"]), + ) + self.assertEqual( + self.product.driver.guess_asset_key_and_roles( + "s3://foo/1/28/0/foo.bar", self.product + ), + ("foo.bar", ["auxiliary"]), + )