Switch to HTTPX to address regression in Python 3.10

setup.cfg

@@ -51,8 +51,7 @@ install_requires =
     importlib-metadata; python_version<"3.8"
     jsonschema>=4.4.0
     protobuf~=3.20
-    urllib3<2
-    requests
+    httpx
     rich
     ruamel.yaml
     tenacity
@@ -106,7 +105,7 @@ testing =
     pytest-asyncio
     pytest-mock
     pytest-timeout
-    requests_mock
+    pytest-httpx
 
 scanner =
     psutil

setup.py

@@ -1,11 +1,12 @@
 """
-    Setup file for ostorlab.
-    Use setup.cfg to configure your project.
+Setup file for ostorlab.
+Use setup.cfg to configure your project.
 
-    This file was generated with PyScaffold 4.1.1.
-    PyScaffold helps you to put up the scaffold of your new Python project.
-    Learn more under: https://pyscaffold.org/
+This file was generated with PyScaffold 4.1.1.
+PyScaffold helps you to put up the scaffold of your new Python project.
+Learn more under: https://pyscaffold.org/
 """
+
 from setuptools import setup
 
 if __name__ == "__main__":

src/ostorlab/__init__.py

@@ -1,4 +1,5 @@
 """Ostorlab main package."""
+
 import logging
 
 import click

src/ostorlab/agent/agent.py

@@ -5,6 +5,7 @@
 
 To use it, check out documentations at https://docs.ostorlab.co/.
 """
+
 import abc
 import argparse
 import asyncio

src/ostorlab/agent/definitions.py

@@ -1,4 +1,5 @@
 """Agent and Agent group definitions and settings dataclasses."""
+
 import dataclasses
 import io
 from typing import List, Optional, Dict, Any

src/ostorlab/agent/kb/kb.py

@@ -1,4 +1,5 @@
 """KB provides automatic mapping from the KB entry folder name to the meta content."""
+
 import dataclasses
 import json
 import pathlib

src/ostorlab/agent/message/message.py

@@ -21,6 +21,7 @@
 }
 
 """
+
 import dataclasses
 from typing import Any, Dict
 

src/ostorlab/agent/message/proto_dict.py

@@ -4,7 +4,8 @@
 
 This code is credited to protobuf-to-dict. The app is written and maintained by Ben Hodgson, with significant
 contributions from Nino Walker, Jonathan Klaassen, and Tristram Gräbener.
- """
+"""
+
 from google.protobuf.descriptor import FieldDescriptor
 from ostorlab import exceptions
 from typing import Dict, Callable, Any

src/ostorlab/agent/message/serializer.py

@@ -1,4 +1,5 @@
 """Serializer handles matching of selector to the proper protobuf message definition."""
+
 import importlib
 import logging
 import os

src/ostorlab/agent/mixins/agent_healthcheck_mixin.py

@@ -11,6 +11,7 @@
     status_agent.start()
 ```
 """
+
 import logging
 from threading import Thread
 from typing import Optional, Callable, List

src/ostorlab/agent/mixins/agent_mq_mixin.py

@@ -46,9 +46,9 @@ def __init__(
         self._max_priority = max_priority
         self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=3)
         self._queue: Optional[aio_pika.Queue] = None
-        self._connection_pool: aio_pika.pool.Pool[
-            aio_pika.Connection
-        ] = aio_pika.pool.Pool(self._get_connection, max_size=32, loop=self._loop)
+        self._connection_pool: aio_pika.pool.Pool[aio_pika.Connection] = (
+            aio_pika.pool.Pool(self._get_connection, max_size=32, loop=self._loop)
+        )
         self._channel_pool: aio_pika.pool.Pool[aio_pika.Channel] = aio_pika.pool.Pool(
             self._get_channel, max_size=64, loop=self._loop
         )

src/ostorlab/agent/mixins/agent_open_telemetry_mixin.py

@@ -3,6 +3,7 @@
 The mixin overrides the behaviour of the main methods of the agent, mainly emit and process to send traces,
 metadata, metrics and exceptions.
 """
+
 import base64
 import io
 import json
@@ -96,9 +97,9 @@ def _get_gcp_exporter(self, parsed_url) -> cloud_trace.CloudTraceSpanExporter:
             )
             service_account_key_temp_file.write(service_account_json_content)
             # the env variable GOOGLE_APPLICATION_CREDENTIALS points to a file defining the service account credentials
-            os.environ[
-                "GOOGLE_APPLICATION_CREDENTIALS"
-            ] = service_account_key_temp_file.name
+            os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = (
+                service_account_key_temp_file.name
+            )
         return cloud_trace.CloudTraceSpanExporter(project_id=project_id)
 
 

src/ostorlab/agent/mixins/agent_persist_mixin.py

@@ -11,6 +11,7 @@
     is_new = not status_agent.set_is_member()
 ```
 """
+
 import ipaddress
 import logging
 from typing import Dict, Set, Callable, Optional, Union

src/ostorlab/agent/mixins/agent_report_vulnerability_mixin.py

@@ -3,6 +3,7 @@
 Definition of the main method to process an entry from the KB - Knowledge Base,
 and emit a vulnerability message.
 """
+
 import dataclasses
 import enum
 from typing import Optional, List, Any, Union, Dict

src/ostorlab/agent/mixins/protocols/emit.py

@@ -1,4 +1,5 @@
 """Definition of the emitting protocol exposing the emit method."""
+
 from typing import Any, Dict, Protocol, Optional
 
 

src/ostorlab/agent/schema/loader.py

@@ -1,4 +1,5 @@
 """Agent and Agent Group loader helper methods that handles schema validation and safe loading."""
+
 import io
 import pathlib
 

src/ostorlab/agent/schema/validator.py

@@ -1,9 +1,10 @@
 """Validation of Yaml configuration files against json schema files.
 
-    Typical usage example:
-    validator = Validator(json_schema_file_object)
-    validator.validate(yaml_file_object)
+Typical usage example:
+validator = Validator(json_schema_file_object)
+validator.validate(yaml_file_object)
 """
+
 import io
 import json
 

src/ostorlab/apis/add_scanner_state.py

@@ -1,4 +1,5 @@
 """Add scanner state via an API Request."""
+
 from typing import Dict, Optional
 import json
 

src/ostorlab/apis/agent_group.py

@@ -1,4 +1,5 @@
 """Create Agent Group via an API Request."""
+
 from typing import Dict, List, Optional, Union
 
 from ostorlab.apis import request

src/ostorlab/apis/assets.py

@@ -1,4 +1,5 @@
 """Create Asset via an API Request."""
+
 import json
 from typing import Dict, Optional
 

src/ostorlab/apis/create_agent_scan.py

@@ -1,4 +1,5 @@
 """Create Scan via an API Request."""
+
 import json
 from typing import Dict, Optional
 

src/ostorlab/apis/request.py

@@ -1,4 +1,5 @@
 """Module responsible for preparing an API request."""
+
 import abc
 import logging
 from typing import Dict, Optional

src/ostorlab/apis/runners/authenticated_runner.py

@@ -1,16 +1,17 @@
 """Handles all authenticated API calls and behind the scenes operations such as authentication, validation, etc.
 
-    Typical usage example:
+Typical usage example:
 
-    authenticated_runner = AuthenticatedAPIRunner(username, password, token_duration)
-    authenticated_runner.authenticate()
+authenticated_runner = AuthenticatedAPIRunner(username, password, token_duration)
+authenticated_runner.authenticate()
 """
+
 import datetime
 import logging
 from typing import Dict, Optional, Any
 
 import click
-import requests
+import httpx
 import ubjson
 import json
 
@@ -161,22 +162,16 @@ def execute(self, request: api_request.APIRequest) -> Dict[str, Any]:
 
     def _sent_request(
         self, request: api_request.APIRequest, headers: Optional[Dict[str, str]] = None
-    ) -> requests.Response:
+    ) -> httpx.Response:
         """Sends an API request."""
-        if self._proxy is not None:
-            proxy = {"https": self._proxy}
-        else:
-            proxy = None
-
-        return requests.post(
-            self.endpoint,
-            data=request.data,
-            files=request.files,
-            headers=headers,
-            proxies=proxy,
-            verify=self._verify,
-            timeout=runner.REQUEST_TIMEOUT,
-        )
+        with httpx.Client(proxy=self._proxy, verify=self._verify) as client:
+            return client.post(
+                self.endpoint,
+                data=request.data,
+                files=request.files,
+                headers=headers,
+                timeout=runner.REQUEST_TIMEOUT,
+            )
 
     def execute_ubjson_request(self, request: api_request.APIRequest) -> Dict[str, Any]:
         """Executes a request using the Authenticated GraphQL API.
@@ -219,19 +214,13 @@ def execute_ubjson_request(self, request: api_request.APIRequest) -> Dict[str, A
 
     def _send_ubjson_request(
         self, request: api_request.APIRequest, headers: Optional[Dict[str, str]] = None
-    ) -> requests.Response:
+    ) -> httpx.Response:
         """Sends an API request."""
-        if self._proxy is not None:
-            proxy = {"https": self._proxy}
-        else:
-            proxy = None
-
-        return requests.post(
-            self.endpoint,
-            data=ubjson.dumpb(request.data),
-            files=request.files,
-            headers=headers,
-            proxies=proxy,
-            verify=self._verify,
-            timeout=runner.REQUEST_TIMEOUT,
-        )
+        with httpx.Client(proxy=self._proxy, verify=self._verify) as client:
+            return client.post(
+                self.endpoint,
+                data=ubjson.dumpb(request.data),
+                files=request.files,
+                headers=headers,
+                timeout=runner.REQUEST_TIMEOUT,
+            )

src/ostorlab/apis/runners/login_runner.py

@@ -1,11 +1,11 @@
 """Handles calls to the token API.
 
-    Typical usage example:
-    public_runner = LoginAPIRunner(username, password, otp_token)
-    public_runner.login_user()
+Typical usage example:
+public_runner = LoginAPIRunner(username, password, otp_token)
+public_runner.login_user()
 """
 
-import requests
+import httpx
 from typing import Dict, Optional, Any
 
 from ostorlab.apis.runners import runner
@@ -45,7 +45,7 @@ def endpoint(self) -> str:
         """Token API endpoint."""
         return TOKEN_ENDPOINT
 
-    def login_user(self) -> requests.models.Response:
+    def login_user(self) -> httpx.Response:
         """Logs in the user.
 
         Returns:

src/ostorlab/apis/runners/public_runner.py

@@ -1,13 +1,13 @@
 """Handles the public API calls.
 
-    Typical usage example:
-    public_runner = PublicAPIRunner()
-    public_runner.execute()
+Typical usage example:
+public_runner = PublicAPIRunner()
+public_runner.execute()
 """
 
 from typing import Dict, Any, Optional
 
-import requests
+import httpx
 
 from ostorlab.apis import request as api_request
 from ostorlab.apis.runners import runner
@@ -50,16 +50,11 @@ def execute(self, request: api_request.APIRequest) -> Dict[str, Any]:
 
     def _sent_request(
         self, request: api_request.APIRequest, headers: Optional[Dict[str, str]] = None
-    ) -> requests.Response:
+    ) -> httpx.Response:
         """Sends an API request."""
-        if self._proxy is not None:
-            proxy = {"https": self._proxy}
-        else:
-            proxy = None
-        return requests.post(
-            self.endpoint,
-            data=request.data,
-            proxies=proxy,
-            verify=self._verify,
-            timeout=runner.REQUEST_TIMEOUT,
-        )
+        with httpx.Client(proxy=self._proxy, verify=self._verify) as client:
+            return client.post(
+                self.endpoint,
+                data=request.data,
+                timeout=runner.REQUEST_TIMEOUT,
+            )

src/ostorlab/apis/runners/runner.py

@@ -6,7 +6,7 @@
 import abc
 from typing import Dict, Optional, Any
 
-import requests
+import httpx
 
 from ostorlab import configuration_manager as config_manager
 from ostorlab.apis import request as api_request
@@ -60,19 +60,13 @@ def execute(self, request: api_request.APIRequest) -> Dict[str, Any]:
 
     def _sent_request(
         self, request: api_request.APIRequest, headers: Optional[Dict[str, str]] = None
-    ) -> requests.Response:
+    ) -> httpx.Response:
         """Sends an API request."""
-        if self._proxy is not None:
-            proxy = {"https": self._proxy}
-        else:
-            proxy = None
-
-        return requests.post(
-            self.endpoint,
-            data=request.data,
-            files=request.files,
-            headers=headers,
-            proxies=proxy,
-            verify=self._verify,
-            timeout=REQUEST_TIMEOUT,
-        )
+        with httpx.Client(proxy=self._proxy, verify=self._verify) as client:
+            return client.post(
+                self.endpoint,
+                data=request.data,
+                files=request.files,
+                headers=headers,
+                timeout=REQUEST_TIMEOUT,
+            )

src/ostorlab/apis/scan_create.py

@@ -1,4 +1,5 @@
 """Create mobile scan API."""
+
 import enum
 import io
 import json