Merge pull request #504 from crim-ca/fix-distutils

CHANGES.rst

@@ -16,6 +16,10 @@ Changes:
 
 Fixes:
 ------
+- Fix ``distutils.version.LooseVersion`` marked for deprecation for upcoming versions.
+  Use ``packaging.version.Version`` substitute whenever possible, but preserve backward
+  compatibility with ``distutils`` in case of older Python not supporting it.
+- Fix ``cli._update_files`` so there are no attempts to upload remote references to the `Vault`.
 - No change.
 
 .. _changes_4.27.0:
@@ -51,7 +55,6 @@ Fixes:
 ------
 - Fix ``cli._update_files`` so there are no attempts to upload remote references to the vault.
 
-
 .. _changes_4.26.0:
 
 `4.26.0 <https://github.com/crim-ca/weaver/tree/4.26.0>`_ (2022-10-31)

Makefile

@@ -68,8 +68,11 @@ PYTHON_VERSION_MINOR := $(shell echo $(PYTHON_VERSION) | cut -f 2 -d '.')
 PYTHON_VERSION_PATCH := $(shell echo $(PYTHON_VERSION) | cut -f 3 -d '.' | cut -f 1 -d ' ')
 PIP_USE_FEATURE := `python -c '\
 	import pip; \
-	from distutils.version import LooseVersion; \
-	print(LooseVersion(pip.__version__) < LooseVersion("21.0"))'`
+	try: \
+		from packaging.version import Version \
+	except ImportError: \
+		from distutils.version import LooseVersion as Version \
+	print(Version(pip.__version__) < Version("21.0"))'`
 PIP_XARGS ?=
 ifeq ("$(PIP_USE_FEATURE)", "True")
   PIP_XARGS := --use-feature=2020-resolver $(PIP_XARGS)

tests/wps_restapi/test_jobs.py

@@ -7,7 +7,6 @@
 import unittest
 import warnings
 from datetime import date
-from distutils.version import LooseVersion
 from typing import TYPE_CHECKING
 
 import colander
@@ -31,6 +30,7 @@
     setup_mongodb_processstore,
     setup_mongodb_servicestore
 )
+from weaver.compat import Version
 from weaver.datatype import Job, Service
 from weaver.execute import ExecuteMode, ExecuteResponse, ExecuteTransmissionMode
 from weaver.formats import ContentType
@@ -208,7 +208,7 @@ def make_job(self,
 
     def get_job_request_auth_mock(self, user_id):
         is_admin = self.user_admin_id == user_id
-        if LooseVersion(get_module_version("pyramid")) >= LooseVersion("2"):
+        if Version(get_module_version("pyramid")) >= Version("2"):
             authn_policy_class = "pyramid.security.SecurityAPIMixin"
             authz_policy_class = "pyramid.security.SecurityAPIMixin"
         else:

tests/wps_restapi/test_providers.py

@@ -1,5 +1,4 @@
 import unittest
-from distutils.version import LooseVersion
 
 import owslib
 import pyramid.testing
@@ -14,6 +13,7 @@
     setup_mongodb_processstore,
     setup_mongodb_servicestore
 )
+from weaver.compat import Version
 from weaver.config import WeaverConfiguration
 from weaver.datatype import Service
 from weaver.execute import ExecuteControlOption, ExecuteTransmissionMode
@@ -290,7 +290,7 @@ def test_get_provider_processes(self):
             remote_processes.append(process["id"])
         assert resources.TEST_REMOTE_SERVER_WPS1_PROCESS_ID in remote_processes
 
-    @pytest.mark.xfail(condition=LooseVersion(owslib.__version__) <= LooseVersion("0.25.0"),
+    @pytest.mark.xfail(condition=Version(owslib.__version__) <= Version("0.25.0"),
                        reason="OWSLib fix for retrieval of processVersion from DescribeProcess not yet available "
                               "(https://github.com/geopython/OWSLib/pull/794)")
     @mocked_remote_server_requests_wps1([
@@ -602,7 +602,7 @@ def test_get_provider_process_literal_values(self):
         assert outputs[1]["formats"][0]["default"] is True
         assert "maximumMegabytes" not in outputs[1]["formats"][0]  # never applies, even with OWSLib update
 
-    @pytest.mark.xfail(condition=LooseVersion(owslib.__version__) <= LooseVersion("0.25.0"),
+    @pytest.mark.xfail(condition=Version(owslib.__version__) <= Version("0.25.0"),
                        reason="OWSLib fix for retrieval of maximumMegabytes from ComplexData not yet available "
                               "(https://github.com/geopython/OWSLib/pull/796)")
     @mocked_remote_server_requests_wps1([

weaver/compat.py

@@ -0,0 +1,40 @@
+from typing import Tuple, Union
+
+try:
+    from packaging.version import InvalidVersion, Version as PackageVersion  # pylint: disable=unused-import
+
+    class Version(PackageVersion):
+        @property
+        def version(self) -> Tuple[Union[int, str], ...]:
+            parts = [part for part in self._version[1:] if part is not None]
+            parts = tuple(part_group for part in parts for part_group in part)
+            return parts
+
+        @property
+        def patch(self):
+            return self.micro
+
+except ImportError:  # pragma: no cover  # for backward compatibility
+    from distutils.version import LooseVersion as BaseVersion  # pylint: disable=deprecated-module
+
+    InvalidVersion = ValueError
+
+    class Version(BaseVersion):
+        @property
+        def major(self) -> int:
+            num = self.version[0:1]
+            return int(num[0]) if num else None
+
+        @property
+        def minor(self) -> int:
+            num = self.version[1:2]
+            return int(num[0]) if num else None
+
+        @property
+        def patch(self) -> int:
+            num = self.version[2:3]
+            return int(num[0]) if num else None
+
+        @property
+        def micro(self) -> int:
+            return self.patch

weaver/processes/utils.py

@@ -4,7 +4,6 @@
 import pathlib
 import warnings
 from copy import deepcopy
-from distutils.version import LooseVersion
 from typing import TYPE_CHECKING
 from urllib.parse import parse_qs, urlparse
 
@@ -22,6 +21,7 @@
 )
 from pyramid.settings import asbool
 
+from weaver.compat import Version
 from weaver.config import (
     WEAVER_CONFIG_DIR,
     WEAVER_DEFAULT_WPS_PROCESSES_CONFIG,
@@ -900,7 +900,7 @@ def register_wps_processes_static(service_url, service_name, service_visibility,
 
     LOGGER.info("Fetching WPS-1: [%s]", service_url)
     wps = get_wps_client(service_url, container)
-    if LooseVersion(wps.version) >= LooseVersion("2.0"):
+    if Version(wps.version) >= Version("2.0"):
         LOGGER.warning("Invalid WPS-1 provider, version was [%s]", wps.version)
         return
     wps_processes = [wps.describeprocess(p) for p in service_processes] or wps.processes

weaver/typedefs.py

@@ -9,12 +9,13 @@
     import typing
     import uuid
     from datetime import datetime
-    from distutils.version import LooseVersion
     from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Type, TypeVar, Union
 
     import psutil
     from typing_extensions import Literal, NotRequired, ParamSpec, Protocol, Required, TypeAlias, TypedDict
 
+    from weaver.compat import Version
+
     if hasattr(os, "PathLike"):
         FileSystemPathType = Union[os.PathLike, str]
     else:
@@ -83,7 +84,7 @@
     AnyValueType = Optional[ValueType]  # avoid naming ambiguity with PyWPS AnyValue
     AnyKey = Union[str, int]
     AnyUUID = Union[str, uuid.UUID]
-    AnyVersion = Union[LooseVersion, Number, str, Tuple[int, ...], List[int]]
+    AnyVersion = Union[Version, Number, str, Tuple[int, ...], List[int]]
     # add more levels of explicit definitions than necessary to simulate JSON recursive structure better than 'Any'
     # amount of repeated equivalent definition makes typing analysis 'work well enough' for most use cases
     _JSON: TypeAlias = "JSON"

weaver/utils.py

@@ -17,7 +17,6 @@
 from concurrent.futures import ALL_COMPLETED, CancelledError, ThreadPoolExecutor, as_completed, wait as wait_until
 from copy import deepcopy
 from datetime import datetime
-from distutils.version import LooseVersion
 from typing import TYPE_CHECKING, overload
 from urllib.parse import ParseResult, unquote, urlparse, urlunsplit
 
@@ -55,14 +54,14 @@
 from werkzeug.wrappers import Request as WerkzeugRequest
 from yaml.scanner import ScannerError
 
-import xml_util
 from weaver.base import Constants, ExtendedEnum
+from weaver.compat import Version
 from weaver.exceptions import WeaverException
 from weaver.execute import ExecuteControlOption, ExecuteMode
 from weaver.formats import ContentType, get_content_type, repr_json
 from weaver.status import map_status
 from weaver.warning import TimeZoneInfoAlreadySetWarning
-from weaver.xml_util import XML
+from weaver.xml_util import HTML_TREE_BUILDER, XML
 
 if TYPE_CHECKING:
     from types import FrameType
@@ -704,7 +703,7 @@ def is_valid_url(url):
     # type: (Optional[str]) -> TypeGuard[str]
     try:
         return bool(urlparse(url).scheme)
-    except Exception:  # noqa: W0703 # nosec: B110
+    except (TypeError, ValueError):
         return False
 
 
@@ -715,14 +714,14 @@ class VersionLevel(Constants):
 
 
 class VersionFormat(Constants):
-    OBJECT = "object"  # LooseVersion
+    OBJECT = "object"  # Version
     STRING = "string"  # "x.y.z"
     PARTS = "parts"    # tuple/list
 
 
 @overload
 def as_version_major_minor_patch(version, version_format):
-    # type: (AnyVersion, Literal[VersionFormat.OBJECT]) -> LooseVersion
+    # type: (AnyVersion, Literal[VersionFormat.OBJECT]) -> Version
     ...
 
 
@@ -750,7 +749,7 @@ def as_version_major_minor_patch(version, version_format=VersionFormat.PARTS):
     Generates a ``MAJOR.MINOR.PATCH`` version with padded with zeros for any missing parts.
     """
     if isinstance(version, (str, float, int)):
-        ver_parts = list(LooseVersion(str(version)).version)
+        ver_parts = list(Version(str(version)).version)
     elif isinstance(version, (list, tuple)):
         ver_parts = [int(part) for part in version]
     else:
@@ -761,7 +760,7 @@ def as_version_major_minor_patch(version, version_format=VersionFormat.PARTS):
         ver_str = ".".join(str(part) for part in ver_tuple)
         if version_format == VersionFormat.STRING:
             return ver_str
-        return LooseVersion(ver_str)
+        return Version(ver_str)
     return ver_tuple
 
 
@@ -870,7 +869,7 @@ def as_int(value, default):
     """
     try:
         return int(value)
-    except Exception:  # noqa: W0703 # nosec: B110
+    except (OverflowError, TypeError, ValueError):
         pass
     return default
 
@@ -1132,13 +1131,13 @@ def pass_http_error(exception, expected_http_error):
     """
     Silently ignore a raised HTTP error that matches the specified error code of the reference exception class.
 
-    Given an `HTTPError` of any type (:mod:`pyramid`, :mod:`requests`), ignores the exception if the actual
+    Given an :class:`HTTPError` of any type (:mod:`pyramid`, :mod:`requests`), ignores the exception if the actual
     error matches the status code. Other exceptions are re-raised.
     This is equivalent to capturing a specific ``Exception`` within an ``except`` block and calling ``pass`` to drop it.
 
-    :param exception: any `Exception` instance ("object" from a `try..except exception as "object"` block).
-    :param expected_http_error: single or list of specific pyramid `HTTPError` to handle and ignore.
-    :raise exception: if it doesn't match the status code or is not an `HTTPError` of any module.
+    :param exception: Any :class:`Exception` instance.
+    :param expected_http_error: Single or list of specific pyramid `HTTPError` to handle and ignore.
+    :raise exception: If it doesn't match the status code or is not an `HTTPError` of any module.
     """
     if not hasattr(expected_http_error, "__iter__"):
         expected_http_error = [expected_http_error]
@@ -1421,8 +1420,8 @@ def get_ssl_verify_option(method, url, settings, request_options=None):
 
     :param method: request method (GET, POST, etc.).
     :param url: request URL.
-    :param settings: application setting container with pre-loaded *request options* specifications.
-    :param request_options: pre-processed *request options* for method/URL to avoid re-parsing the settings.
+    :param settings: application setting container with preloaded *request options* specifications.
+    :param request_options: preprocessed *request options* for method/URL to avoid parsing the settings again.
     :returns: SSL ``verify`` option to be passed down to some ``request`` function.
     """
     if not settings:
@@ -2582,13 +2581,12 @@ def _list_refs(_url, _data=None):
             _scheme = _url.split("://")[0]
             _opts = options.get(_scheme, {})  # type: ignore
             _resp = request_extra("GET", _url, settings=settings, **_opts, **kwargs)
-            ctype = get_header("Content-Type", _resp.headers, default=ContentType.TEXT_HTML)
-            if _resp.status_code != 200 or not any(
-                _type in ctype for _type in [ContentType.TEXT_HTML] + list(ContentType.ANY_XML)
-            ):
+            _ctype = get_header("Content-Type", _resp.headers, default=ContentType.TEXT_HTML)
+            _xml_like_ctypes = [ContentType.TEXT_HTML] + list(ContentType.ANY_XML)
+            if _resp.status_code != 200 or not any(_type in _ctype for _type in _xml_like_ctypes):
                 return []
             _data = _resp.text
-        _html = BeautifulSoup(_data, builder=xml_util.HTML_TREE_BUILDER)
+        _html = BeautifulSoup(_data, builder=HTML_TREE_BUILDER)
         _href = (_ref.get("href") for _ref in _html.find_all("a", recursive=True))
         _href = filter_directory_forbidden(_href)  # preemptively remove forbidden items, avoid access/download attempts
         for _ref in _href:

weaver/wps_restapi/processes/utils.py

@@ -6,6 +6,7 @@
 from pyramid.httpexceptions import HTTPBadRequest
 from pyramid.settings import asbool
 
+from weaver.compat import InvalidVersion
 from weaver.config import WeaverFeature, get_weaver_configuration
 from weaver.database import get_db
 from weaver.formats import ContentType
@@ -92,7 +93,10 @@ def get_processes_filtered_by_valid_schemas(request):
     invalid_processes_ids = []
     for process in processes:  # type: Process
         try:
-            valid_processes.append(process.summary(revision=with_revisions))
+            try:
+                valid_processes.append(process.summary(revision=with_revisions))
+            except (InvalidVersion, ValueError) as exc:
+                raise colander.Invalid(sd.ProcessSummary, value=None, msg=str(exc))
         except colander.Invalid as invalid:
             process_ref = process.tag if with_revisions else process.identifier
             LOGGER.debug("Invalid process [%s] because:\n%s", process_ref, invalid)

weaver/wps_restapi/swagger_definitions.py

@@ -445,7 +445,6 @@ class ProcessIdentifierTag(AnyOfKeywordSchema):
 
 
 class Version(ExtendedSchemaNode):
-    # note: internally use LooseVersion, so don't be too strict about pattern
     schema_type = String
     description = "Version string."
     example = "1.2.3"