From 77494191df806f0b82f037c4e72c22c213dbce57 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Tue, 23 Apr 2024 10:48:01 +0200 Subject: [PATCH 1/7] refactor: remove unused AwsSearch --- eodag/plugins/search/qssearch.py | 28 ---------------------------- setup.cfg | 1 - 2 files changed, 29 deletions(-) diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index 9e8c5a2df..33ef0ef0d 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -1002,34 +1002,6 @@ def _request( return response -class AwsSearch(QueryStringSearch): - """A specialisation of RestoSearch that modifies the way the EOProducts are built - from the search results""" - - def normalize_results( - self, results: List[Dict[str, Any]], **kwargs: Any - ) -> List[EOProduct]: - """Transform metadata from provider representation to eodag representation""" - normalized: List[EOProduct] = [] - logger.debug("Adapting plugin results to eodag product representation") - for result in results: - ref = result["properties"]["title"].split("_")[5] - year = result["properties"]["completionDate"][0:4] - month = str(int(result["properties"]["completionDate"][5:7])) - day = str(int(result["properties"]["completionDate"][8:10])) - - properties = QueryStringSearch.extract_properties[self.config.result_type]( - result, self.get_metadata_mapping(kwargs.get("productType")) - ) - - properties["downloadLink"] = ( - "s3://tiles/{ref[1]}{ref[2]}/{ref[3]}/{ref[4]}{ref[5]}/{year}/" - "{month}/{day}/0/" - ).format(**locals()) - normalized.append(EOProduct(self.provider, properties, **kwargs)) - return normalized - - class ODataV4Search(QueryStringSearch): """A specialisation of a QueryStringSearch that does a two step search to retrieve all products metadata""" diff --git a/setup.cfg b/setup.cfg index 6c46acdee..1155659ca 100644 --- a/setup.cfg +++ b/setup.cfg @@ -153,7 +153,6 @@ eodag.plugins.download = eodag.plugins.search = CSWSearch = eodag.plugins.search.csw:CSWSearch QueryStringSearch = eodag.plugins.search.qssearch:QueryStringSearch - AwsSearch = eodag.plugins.search.qssearch:AwsSearch ODataV4Search = eodag.plugins.search.qssearch:ODataV4Search PostJsonSearch = eodag.plugins.search.qssearch:PostJsonSearch StacSearch = eodag.plugins.search.qssearch:StacSearch From 5954068646edeaf70b142799b4387dc495c167f9 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Tue, 23 Apr 2024 10:54:46 +0200 Subject: [PATCH 2/7] build: refactor and add new optional dependencies --- setup.cfg | 95 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 54 insertions(+), 41 deletions(-) diff --git a/setup.cfg b/setup.cfg index 1155659ca..db9c8d681 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,40 +37,64 @@ packages = find: include_package_data = True python_requires = >=3.6 install_requires = + annotated-types click - requests - urllib3 - python-dateutil - PyYAML - tqdm - shapely - pyshp - OWSLib >=0.27.1 + geojson + jsonpath-ng < 1.6.0 + lxml orjson < 3.10.0;python_version>='3.12' and platform_system=='Windows' orjson;python_version<'3.12' and platform_system!='Windows' - geojson + pydantic >= 2.1.0 + pydantic_core pyproj >= 2.1.0 - usgs >= 0.3.1 + pyshp + pystac >= 1.0.0b1 + python-dateutil + PyYAML + requests + setuptools + shapely + stream-zip + tqdm + typing_extensions + urllib3 + Whoosh + +[options.extras_require] +all = + eodag[all-providers,csw,server,tutorials] +all-providers = + eodag[aws,ecmwf,usgs] +aws = boto3 botocore +csw = + OWSLib >=0.27.1 +ecmwf = + ecmwf-api-client +usgs = + usgs >= 0.3.1 +server = fastapi >= 0.93.0 + pygeofilter starlette uvicorn - jsonpath-ng < 1.6.0 - lxml - Whoosh - pystac >= 1.0.0b1 - ecmwf-api-client - stream-zip - pydantic >= 2.1.0 - pydantic_core - typing_extensions - annotated-types - setuptools - pygeofilter -[options.extras_require] +notebook = tqdm[notebook] +tutorials = + eodag[notebook] + eodag-cube >= 0.2.0 + jupyter + ipyleaflet >= 0.10.0 + ipywidgets + matplotlib + folium + imageio + rasterio + netcdf4 + dev = + eodag[all-providers,csw,server] pytest pytest-cov # pytest-html max version set and py requirement added @@ -92,19 +116,8 @@ dev = stdlib-list boto3-stubs[essential] types-lxml - -notebook = tqdm[notebook] -tutorials = - eodag-cube >= 0.2.0 - jupyter - ipyleaflet >= 0.10.0 - ipywidgets - matplotlib - folium - imageio - rasterio - netcdf4 docs = + eodag[all] sphinx sphinx-book-theme sphinx-copybutton @@ -127,8 +140,8 @@ exclude = console_scripts = eodag = eodag.cli:eodag eodag.plugins.api = - UsgsApi = eodag.plugins.apis.usgs:UsgsApi - EcmwfApi = eodag.plugins.apis.ecmwf:EcmwfApi + UsgsApi = eodag.plugins.apis.usgs:UsgsApi [usgs] + EcmwfApi = eodag.plugins.apis.ecmwf:EcmwfApi [ecmwf] eodag.plugins.auth = GenericAuth = eodag.plugins.authentication.generic:GenericAuth HTTPHeaderAuth = eodag.plugins.authentication.header:HTTPHeaderAuth @@ -146,12 +159,12 @@ eodag.plugins.crunch = FilterProperty = eodag.plugins.crunch.filter_property:FilterProperty FilterDate = eodag.plugins.crunch.filter_date:FilterDate eodag.plugins.download = - AwsDownload = eodag.plugins.download.aws:AwsDownload + AwsDownload = eodag.plugins.download.aws:AwsDownload [aws] HTTPDownload = eodag.plugins.download.http:HTTPDownload S3RestDownload = eodag.plugins.download.s3rest:S3RestDownload - CreodiasS3Download = eodag.plugins.download.creodias_s3:CreodiasS3Download + CreodiasS3Download = eodag.plugins.download.creodias_s3:CreodiasS3Download [aws] eodag.plugins.search = - CSWSearch = eodag.plugins.search.csw:CSWSearch + CSWSearch = eodag.plugins.search.csw:CSWSearch [csw] QueryStringSearch = eodag.plugins.search.qssearch:QueryStringSearch ODataV4Search = eodag.plugins.search.qssearch:ODataV4Search PostJsonSearch = eodag.plugins.search.qssearch:PostJsonSearch @@ -160,7 +173,7 @@ eodag.plugins.search = BuildSearchResult = eodag.plugins.search.build_search_result:BuildSearchResult BuildPostSearchResult = eodag.plugins.search.build_search_result:BuildPostSearchResult DataRequestSearch = eodag.plugins.search.data_request_search:DataRequestSearch - CreodiasS3Search = eodag.plugins.search.creodias_s3:CreodiasS3Search + CreodiasS3Search = eodag.plugins.search.creodias_s3:CreodiasS3Search [aws] [flake8] ignore = E203, W503 From 001ecd129c0a25d6f99c2346a47aabd6cd2c84ae Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Tue, 23 Apr 2024 11:23:31 +0200 Subject: [PATCH 3/7] feat: skip plugins needing missing deps --- eodag/api/core.py | 15 +++++++++++++++ eodag/plugins/manager.py | 10 ++++++++++ 2 files changed, 25 insertions(+) diff --git a/eodag/api/core.py b/eodag/api/core.py index fbe4d14ef..081b56db8 100644 --- a/eodag/api/core.py +++ b/eodag/api/core.py @@ -38,6 +38,7 @@ from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypath from eodag.api.search_result import SearchResult from eodag.config import ( + PluginConfig, SimpleYamlProxyConfig, get_ext_product_types_conf, load_default_config, @@ -403,6 +404,20 @@ def _prune_providers_list(self) -> None: for provider in list(self.providers_config.keys()): conf = self.providers_config[provider] + # remove providers using skipped plugins + if [ + v + for v in conf.__dict__.values() + if isinstance(v, PluginConfig) + and getattr(v, "type", None) in self._plugins_manager.skipped_plugins + ]: + self.providers_config.pop(provider) + logger.debug( + f"{provider}: provider needing unavailable plugin has been removed" + ) + continue + + # check authentication if hasattr(conf, "api") and getattr(conf.api, "need_auth", False): credentials_exist = any( [ diff --git a/eodag/plugins/manager.py b/eodag/plugins/manager.py index 856f51ece..5d129ad96 100644 --- a/eodag/plugins/manager.py +++ b/eodag/plugins/manager.py @@ -75,7 +75,10 @@ class PluginManager: product_type_to_provider_config_map: Dict[str, List[ProviderConfig]] + skipped_plugins: List[str] + def __init__(self, providers_config: Dict[str, ProviderConfig]) -> None: + self.skipped_plugins = [] self.providers_config = providers_config # Load all the plugins. This will make all plugin classes of a particular # type to be available in the base plugin class's 'plugins' attribute. @@ -92,6 +95,13 @@ def __init__(self, providers_config: Dict[str, ProviderConfig]) -> None: ): try: entry_point.load() + except pkg_resources.DistributionNotFound: + logger.debug( + "%s plugin skipped, eodag[%s] or eodag[all] needed", + entry_point.name, + ",".join(entry_point.extras), + ) + self.skipped_plugins.append(entry_point.name) except ImportError: import traceback as tb From 0b10c5bd4db16322b53c899fcd3ca399ea4f6d2b Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Tue, 23 Apr 2024 11:28:12 +0200 Subject: [PATCH 4/7] feat: handle missing server deps --- eodag/rest/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eodag/rest/__init__.py b/eodag/rest/__init__.py index 7c2da36d1..4a740d4f6 100644 --- a/eodag/rest/__init__.py +++ b/eodag/rest/__init__.py @@ -16,3 +16,9 @@ # See the License for the specific language governing permissions and # limitations under the License. """EODAG REST API""" +try: + from fastapi import __version__ # noqa: F401 +except ImportError: + raise ImportError( + f"{__name__} not available, please install eodag[server] or eodag[all]" + ) From f43c854c94771778a94dce36e4b10007c8e92213 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Tue, 23 Apr 2024 15:04:12 +0200 Subject: [PATCH 5/7] test: refactor and new optional dependencies --- tests/test_requirements.py | 79 ++++++++++++++++++++++++++++++++++++-- tests/units/test_core.py | 23 ++++++++++- 2 files changed, 97 insertions(+), 5 deletions(-) diff --git a/tests/test_requirements.py b/tests/test_requirements.py index 86e155153..9ef2dc9bb 100644 --- a/tests/test_requirements.py +++ b/tests/test_requirements.py @@ -19,12 +19,15 @@ import ast import configparser import os +import re import unittest +from typing import Any, Dict, Iterator, Set import importlib_metadata from packaging.requirements import Requirement from stdlib_list import stdlib_list +from eodag.config import PluginConfig, load_default_config from tests.context import MisconfiguredError project_path = "./eodag" @@ -32,7 +35,7 @@ allowed_missing_imports = ["eodag"] -def get_imports(filepath): +def get_imports(filepath: str) -> Iterator[Any]: """Get python imports from the given file path""" with open(filepath, "r") as file: try: @@ -55,7 +58,7 @@ def get_imports(filepath): yield node.module.split(".")[0] -def get_project_imports(project_path): +def get_project_imports(project_path: str) -> Set[str]: """Get python imports from the project path""" imports = set() for dirpath, dirs, files in os.walk(project_path): @@ -66,7 +69,7 @@ def get_project_imports(project_path): return imports -def get_setup_requires(setup_cfg_path): +def get_setup_requires(setup_cfg_path: str): """Get requirements from the given setup.cfg file path""" config = configparser.ConfigParser() config.read(setup_cfg_path) @@ -79,12 +82,59 @@ def get_setup_requires(setup_cfg_path): ) +def get_optional_dependencies(setup_cfg_path: str, extra: str) -> Set[str]: + """Get extra requirements from the given setup.cfg file path""" + config = configparser.ConfigParser() + config.read(setup_cfg_path) + deps = set() + for req in config["options.extras_require"][extra].split("\n"): + if req.startswith("eodag["): + for found_extra in re.findall(r"([\w-]+)[,\]]", req): + deps.update(get_optional_dependencies(setup_cfg_path, found_extra)) + elif req: + deps.add(Requirement(req).name) + + return deps + + +def get_resulting_extras(setup_cfg_path: str, extra: str) -> Set[str]: + """Get resulting extras for a single extra from the given setup.cfg file path""" + config = configparser.ConfigParser() + config.read(setup_cfg_path) + extras = set() + for req in config["options.extras_require"][extra].split("\n"): + if req.startswith("eodag["): + extras.update(re.findall(r"([\w-]+)[,\]]", req)) + return extras + + +def get_entrypoints_extras(setup_cfg_path: str) -> Dict[str, str]: + """Get entrypoints and associated extra from the given setup.cfg file path""" + config = configparser.ConfigParser() + config.read(setup_cfg_path) + plugins_extras_dict = dict() + for group in config["options.entry_points"].keys(): + for ep in config["options.entry_points"][group].split("\n"): + # plugin entrypoint with associated extra + match = re.search(r"^(\w+) = [\w\.:]+ \[(\w+)\]$", ep) + if match: + plugins_extras_dict[match.group(1)] = match.group(2) + continue + # plugin entrypoint without extra + match = re.search(r"^(\w+) = [\w\.:]+$", ep) + if match: + plugins_extras_dict[match.group(1)] = None + + return plugins_extras_dict + + class TestRequirements(unittest.TestCase): - def test_requirements(self): + def test_all_requirements(self): """Needed libraries must be in project requirements""" project_imports = get_project_imports(project_path) setup_requires = get_setup_requires(setup_cfg_path) + setup_requires.update(get_optional_dependencies(setup_cfg_path, "all")) import_required_dict = importlib_metadata.packages_distributions() default_libs = stdlib_list() @@ -102,3 +152,24 @@ def test_requirements(self): 0, f"The following libraries were not found in project requirements: {missing_imports}", ) + + def test_plugins_extras(self): + """All optional dependencies needed by providers must be resolved with all-providers extra""" + + plugins_extras_dict = get_entrypoints_extras(setup_cfg_path) + all_providers_extras = get_resulting_extras(setup_cfg_path, "all-providers") + + providers_config = load_default_config() + plugins = set() + for provider_conf in providers_config.values(): + plugins.update( + [ + getattr(provider_conf, x).type + for x in dir(provider_conf) + if isinstance(getattr(provider_conf, x), PluginConfig) + ] + ) + + for plugin in plugins: + if extra := plugins_extras_dict.get(plugin): + self.assertIn(extra, all_providers_extras) diff --git a/tests/units/test_core.py b/tests/units/test_core.py index 9d12a2ed7..8931105f9 100644 --- a/tests/units/test_core.py +++ b/tests/units/test_core.py @@ -29,7 +29,7 @@ from tempfile import TemporaryDirectory from unittest.mock import Mock -from pkg_resources import resource_filename +from pkg_resources import DistributionNotFound, resource_filename from shapely import wkt from shapely.geometry import LineString, MultiPolygon, Polygon @@ -976,6 +976,27 @@ def test_prune_providers_list(self): os.environ.pop("EODAG__PEPS__SEARCH__NEED_AUTH", None) os.environ.pop("EODAG__PEPS__AUTH__CREDENTIALS__USERNAME", None) + @mock.patch("eodag.plugins.manager.pkg_resources.iter_entry_points", autospec=True) + def test_prune_providers_list_skipped_plugin(self, mock_iter_ep): + """Providers needing skipped plugin must be pruned on init""" + empty_conf_file = resource_filename( + "eodag", os.path.join("resources", "user_conf_template.yml") + ) + + def skip_qssearch(topic): + ep = mock.MagicMock() + if topic == "eodag.plugins.search": + ep.name = "QueryStringSearch" + ep.load = mock.MagicMock(side_effect=DistributionNotFound()) + return [ep] + + mock_iter_ep.side_effect = skip_qssearch + + dag = EODataAccessGateway(user_conf_file_path=empty_conf_file) + self.assertNotIn("peps", dag.available_providers()) + self.assertEqual(dag._plugins_manager.skipped_plugins, ["QueryStringSearch"]) + dag._plugins_manager.skipped_plugins = [] + def test_prune_providers_list_for_search_without_auth(self): """Providers needing auth for search but without auth plugin must be pruned on init""" empty_conf_file = resource_filename( From 8fbac1238faa636882500c3ee5230c750941b103 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Wed, 24 Apr 2024 15:08:56 +0200 Subject: [PATCH 6/7] docs: updated plugins ref --- docs/plugins_reference/auth.rst | 1 + docs/plugins_reference/download.rst | 1 + docs/plugins_reference/search.rst | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/plugins_reference/auth.rst b/docs/plugins_reference/auth.rst index 1a6cb2de4..baade6117 100644 --- a/docs/plugins_reference/auth.rst +++ b/docs/plugins_reference/auth.rst @@ -22,3 +22,4 @@ This table lists all the authentication plugins currently available: eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth eodag.plugins.authentication.keycloak.KeycloakOIDCPasswordAuth eodag.plugins.authentication.qsauth.HttpQueryStringAuth + eodag.plugins.authentication.sas_auth.SASAuth diff --git a/docs/plugins_reference/download.rst b/docs/plugins_reference/download.rst index 7d9c22051..a76af586e 100644 --- a/docs/plugins_reference/download.rst +++ b/docs/plugins_reference/download.rst @@ -17,6 +17,7 @@ This table lists all the download plugins currently available: eodag.plugins.download.http.HTTPDownload eodag.plugins.download.aws.AwsDownload eodag.plugins.download.s3rest.S3RestDownload + eodag.plugins.download.creodias_s3.CreodiasS3Download --------------------------- Download methods call graph diff --git a/docs/plugins_reference/search.rst b/docs/plugins_reference/search.rst index f314e95bc..ba01ff45f 100644 --- a/docs/plugins_reference/search.rst +++ b/docs/plugins_reference/search.rst @@ -15,11 +15,12 @@ This table lists all the search plugins currently available: :toctree: generated/ eodag.plugins.search.qssearch.QueryStringSearch - eodag.plugins.search.qssearch.AwsSearch eodag.plugins.search.qssearch.ODataV4Search eodag.plugins.search.qssearch.PostJsonSearch eodag.plugins.search.qssearch.StacSearch eodag.plugins.search.static_stac_search.StaticStacSearch + eodag.plugins.search.creodias_s3.CreodiasS3Search eodag.plugins.search.build_search_result.BuildSearchResult eodag.plugins.search.build_search_result.BuildPostSearchResult + eodag.plugins.search.data_request_search.DataRequestSearch eodag.plugins.search.csw.CSWSearch From 42c60d1e47901cf7948a21847f24d938c18ce459 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Wed, 24 Apr 2024 15:57:53 +0200 Subject: [PATCH 7/7] docs: updated install instructions --- README.rst | 3 +++ docs/getting_started_guide/install.rst | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 303bbe626..2e09a3c84 100644 --- a/README.rst +++ b/README.rst @@ -80,6 +80,9 @@ And with ``conda`` from the `conda-forge channel `_. + Usage ===== diff --git a/docs/getting_started_guide/install.rst b/docs/getting_started_guide/install.rst index e751e8d88..a8c7a4619 100644 --- a/docs/getting_started_guide/install.rst +++ b/docs/getting_started_guide/install.rst @@ -20,6 +20,20 @@ Or with ``conda`` from the *conda-forge* channel: conda install -c conda-forge eodag +Optional dependencies +^^^^^^^^^^^^^^^^^^^^^ + +Since ``v3.0``, EODAG comes with a minimal set of dependencies. If you want more features, please install using one of +the following extras: + +* ``eodag[all]``, includes everything that would be needed to run EODAG and associated tutorials with all features +* ``eodag[all-providers]``, includes dependencies required to have all providers available +* ``eodag[aws]``, includes dependencies for plugins using Amazon S3 +* ``eodag[csw]``, includes dependencies for plugins using CSW +* ``eodag[ecmwf]``, includes dependencies for :class:`~eodag.plugins.apis.ecmwf.EcmwfApi` (`ecmwf` provider) +* ``eodag[usgs]``, includes dependencies for :class:`~eodag.plugins.apis.usgs.UsgsApi` (`usgs` provider) +* ``eodag[server]``, includes dependencies for server-mode + .. _install_notebooks: Run the notebooks locally @@ -30,7 +44,7 @@ that can be run locally: 1. Install the extras dependencies it requires by executing this command (preferably in a virtual environment):: - python -m pip install eodag[tutorials] + python -m pip install "eodag[tutorials]" 2. Clone ``eodag`` 's repository with git::