From 28cb97489b6e21b012825bbc8e6c213a65897af2 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Thu, 19 Jan 2023 14:31:22 +0100 Subject: [PATCH 01/56] rename http.py to client.py due to causing No Module Found error for http.cookiejar --- src/keboola/http_client/__init__.py | 2 +- src/keboola/http_client/{http.py => client.py} | 2 +- tests/test_http.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename src/keboola/http_client/{http.py => client.py} (99%) diff --git a/src/keboola/http_client/__init__.py b/src/keboola/http_client/__init__.py index 7d9f3d0..71e4886 100644 --- a/src/keboola/http_client/__init__.py +++ b/src/keboola/http_client/__init__.py @@ -1 +1 @@ -from .http import HttpClient # noqa +from .client import HttpClient # noqa diff --git a/src/keboola/http_client/http.py b/src/keboola/http_client/client.py similarity index 99% rename from src/keboola/http_client/http.py rename to src/keboola/http_client/client.py index 8f9e1b8..d465391 100644 --- a/src/keboola/http_client/http.py +++ b/src/keboola/http_client/client.py @@ -6,7 +6,7 @@ import requests from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry +from requests.packages.urllib3.util.retry import Retry # noqa Cookie = Union[Dict[str, str], CookieJar] diff --git a/tests/test_http.py b/tests/test_http.py index 99dc66d..82752b9 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -2,7 +2,7 @@ import urllib.parse as urlparse from unittest.mock import patch -import keboola.http_client.http as client +import keboola.http_client.client as client class TestClientBase(unittest.TestCase): From 9913dfd12ad40981ac7acae8bf5686552ae8db19 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Fri, 20 Jan 2023 17:03:55 +0100 Subject: [PATCH 02/56] httpx initial implementation --- requirements.txt | 3 +- src/keboola/http_client/__init__.py | 1 + src/keboola/http_client/async_client.py | 258 +++++++++++++++++++++++ src/keboola/http_client/retry_wrapper.py | 115 ++++++++++ src/keboola/http_client/test_async.py | 36 ++++ 5 files changed, 412 insertions(+), 1 deletion(-) create mode 100644 src/keboola/http_client/async_client.py create mode 100644 src/keboola/http_client/retry_wrapper.py create mode 100644 src/keboola/http_client/test_async.py diff --git a/requirements.txt b/requirements.txt index 663bd1f..e6954cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -requests \ No newline at end of file +requests +httpx \ No newline at end of file diff --git a/src/keboola/http_client/__init__.py b/src/keboola/http_client/__init__.py index 71e4886..b6ab6da 100644 --- a/src/keboola/http_client/__init__.py +++ b/src/keboola/http_client/__init__.py @@ -1 +1,2 @@ from .client import HttpClient # noqa +from .async_client import AsyncHttpClient diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py new file mode 100644 index 0000000..8026c9d --- /dev/null +++ b/src/keboola/http_client/async_client.py @@ -0,0 +1,258 @@ +import functools +import logging +import urllib.parse as urlparse +from http.cookiejar import CookieJar +from typing import Dict, Union, Tuple, Optional + +from httpx import AsyncClient, AsyncHTTPTransport, Response + +from retry_wrapper import RetryTransport + +# import requests +# from requests.adapters import HTTPAdapter +# from requests.packages.urllib3.util.retry import Retry # noqa + +Cookie = Union[Dict[str, str], CookieJar] + +METHOD_RETRY_WHITELIST = ('GET', 'POST', 'PATCH', 'UPDATE', 'PUT', 'DELETE') +ALLOWED_METHODS = ['GET', 'POST', 'PATCH', 'UPDATE', 'PUT', 'DELETE'] + + +class AsyncHttpClient(): + """ + Base class for implementing a simple HTTP client. Typically used as a base for a REST service client. + + + Usage: + + ```python + from keboola.http_client import HttpClient + + BASE_URL = 'https://connection.keboola.com/v2/storage/' + AUTH_HEADER = { + 'x-storageapi-token': '1234-STORAGETOKENSTRING' + } + DEFAULT_PARAMS = { + 'include': 'columns' + } + DEFAULT_HEADER = { + 'Content-Type': 'application/json' + } + + cl = HttpClient(BASE_URL, default_http_header=DEFAULT_HEADER, + auth_header=AUTH_HEADER, default_params=DEFAULT_PARAMS) + + files_response = cl.get("files", params={"showExpired": True}) + ``` + + """ + + def __init__(self, base_url: str, max_retries: int = 10, backoff_factor: float = 0.3, + status_forcelist: Tuple[int, ...] = (500, 502, 504), default_http_header: Dict = None, + auth_header: Dict = None, auth: Tuple = None, default_params: Dict = None, + allowed_methods: Tuple = METHOD_RETRY_WHITELIST): + """ + Create an endpoint. + + Args: + base_url: The base URL for this endpoint. e.g. https://exampleservice.com/api_v1/ + max_retries: Total number of retries to allow. + backoff_factor: A back-off factor to apply between attempts. + status_forcelist: A set of HTTP status codes that we should force a retry on. e.g. [500,502] + default_http_header: Default header to be sent with each request + eg. ```{ + 'Content-Type' : 'application/json', + 'Accept' : 'application/json' + }``` + auth_header: Auth header to be sent with each request + eg. `{'Authorization': 'Bearer ' + token}` + auth: Default Authentication tuple or object to attach to (from requests.Session().auth). + eg. auth = (user, password) + default_params: default parameters to be sent with each request eg. `{'param':'value'}` + allowed_methods (tuple): Set of upper-cased HTTP method verbs that we should retry on. + """ + if base_url is None: + raise ValueError("Base URL is required.") + # Add trailing slash because of nature of urllib.parse.urljoin() + self.base_url = base_url if base_url.endswith('/') else base_url + '/' + self.max_retries = max_retries + self.backoff_factor = backoff_factor + self.status_forcelist = status_forcelist + self._auth = auth + self._auth_header = auth_header if auth_header else {} + self._default_header = default_http_header if default_http_header else {} + self._default_params = default_params + self.allowed_methods = allowed_methods + + self.s = AsyncClient() + + def _requests_retry_session(self, session=None): + transport = RetryTransport(wrapped_transport=AsyncHTTPTransport(), max_attempts=self.max_retries, + max_backoff_wait=30) + session = session or AsyncClient(transport=transport) + """ + retry = Retry( + total=self.max_retries, + read=self.max_retries, + connect=self.max_retries, + backoff_factor=self.backoff_factor, + status_forcelist=self.status_forcelist, + allowed_methods=self.allowed_methods + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + """ + return session + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False): + # build URL Specification + url_path = str(endpoint_path).strip() if endpoint_path is not None else '' + + if not url_path: + url = self.base_url + elif not is_absolute_path: + url = urlparse.urljoin(str(self.base_url), endpoint_path) + else: + url = endpoint_path + + return url + + async def _request_raw(self, method: str, endpoint_path: Optional[str] = None, **kwargs) -> Response: + """ + Construct a requests call with args and kwargs and process the + results. + + Args: + method: A HTTP method to be used. One of PUT/POST/PATCH/GET/UPDATE/DELETE + endpoint_path (Optional[str]): Optional full URL or a relative URL path. If empty the base_url is used. + **kwargs: Key word arguments to pass to the + [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). + Accepts supported params in requests.sessions.Session#request + eg. params = {'locId':'1'}, header = {some additional header} + parameters and headers are appended to the default ones + ignore_auth - True to skip authentication + is_absolute_path - False to append URL to base url; True to override base url with value of url arg. + + Returns: + A [`requests.Response`](https://requests.readthedocs.io/en/latest/api/#requests.Response) object. + """ + + # build URL Specification + is_absolute_path = kwargs.pop('is_absolute_path', False) + url = self._build_url(endpoint_path, is_absolute_path) + + # Update headers + headers = kwargs.pop('headers', {}) + if headers is None: + headers = {} + + # Default headers + headers.update(self._default_header) + + # Auth headers + if kwargs.pop('ignore_auth', False) is False: + headers.update(self._auth_header) + self.s.headers.update(headers) + self.s.auth = self._auth + + self.s.headers.update(headers) + + # Update parameters + params = kwargs.pop('params', {}) + if params is None: + params = {} + + # Default parameters + if self._default_params is not None: + all_pars = {**params, **self._default_params} + kwargs.update({'params': all_pars}) + + else: + kwargs.update({'params': params}) + + r = await self._requests_retry_session(session=self.s).request(method, url, **kwargs) + return r + + async def get_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, + is_absolute_path: bool = False, cookies: Cookie = None, + ignore_auth: bool = False, **kwargs) -> Response: + """ + Constructs a requests GET call with specified url and kwargs to process the result. + + Args: + endpoint_path: Relative URL path or absolute URL to which the request will be made. + By default a relative path is expected and will be appended to the `self.base_url` value. + + Depending on the value of `is_absolute_path`, the value will be either appended to `self.base_url`, + or used as an absolute URL. + params: Dictionary to send in the query string for the request. + headers: Dictionary of HTTP Headers to send with the request. + is_absolute_path: A boolean value specifying, whether the URL specified in `endpoint_path` parameter + is an absolute path or not. + + If set to False, the value of `endpoint_path` will be appended to `self.base_url` using + [`urllib.parse.urljoin()`](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin) + function. + + If set to True, base url will be overridden and the value of the `endpoint_path` will + used instead. + cookies: Dict or CookieJar object of cookies to send with the request + ignore_auth: Boolean marking, whether the default auth_header should be ignored. + **kwargs: All other keyword arguments supported by + [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). + + Returns: + A [`requests.Response`](https://requests.readthedocs.io/en/latest/api/#requests.Response) object. + """ + + method = 'GET' + r = await self._request_raw(method, endpoint_path, params=params, headers=headers, cookies=cookies, + is_absolute_path=is_absolute_path, ignore_auth=ignore_auth, **kwargs) + return r + + async def get(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, + is_absolute_path: bool = False, cookies: Cookie = None, + ignore_auth: bool = False, **kwargs) -> Response: + """ + Constructs a requests GET call with specified url and kwargs to process the result. + + Args: + endpoint_path: Relative URL path or absolute URL to which the request will be made. + By default a relative path is expected and will be appended to the `self.base_url` value. + + Depending on the value of `is_absolute_path`, the value will be either appended to `self.base_url`, + or used as an absolute URL. + params: Dictionary to send in the query string for the request. + headers: Dictionary of HTTP Headers to send with the request. + is_absolute_path: A boolean value specifying, whether the URL specified in `endpoint_path` parameter + is an absolute path or not. + + If set to False, the value of `endpoint_path` will be appended to `self.base_url` using + [`urllib.parse.urljoin()`](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin) + function. + + If set to True, base url will be overridden and the value of the `endpoint_path` will + used instead. + cookies: Dict or CookieJar object of cookies to send with the request + files: Dictionary of 'name': file-like-objects (or {'name': file-tuple}) for multipart encoding upload. + ignore_auth: Boolean marking, whether the default auth_header should be ignored. + **kwargs: All other keyword arguments supported by + [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). + + Returns: + A JSON-encoded response of the request. + + Raises: + requests.HTTPError: If the API request fails. + """ + + r = await self.get_raw(endpoint_path, params=params, headers=headers, cookies=cookies, + is_absolute_path=is_absolute_path, ignore_auth=ignore_auth, **kwargs) + return r diff --git a/src/keboola/http_client/retry_wrapper.py b/src/keboola/http_client/retry_wrapper.py new file mode 100644 index 0000000..bc19900 --- /dev/null +++ b/src/keboola/http_client/retry_wrapper.py @@ -0,0 +1,115 @@ +from datetime import datetime +import random +from time import sleep + +import httpx +from typing import Union, Iterable, Mapping + +from dateutil.parser import isoparse + + +class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport): + """ + Credits to: https://github.com/matt-mercer + """ + + RETRYABLE_METHODS = frozenset( + ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"] + ) + RETRYABLE_STATUS_CODES = frozenset([413, 429, 503, 504]) + + MAX_BACKOFF_WAIT = 60 + + def __init__( + self, + wrapped_transport: Union[httpx.BaseTransport, httpx.AsyncBaseTransport], + max_attempts: int = 10, + max_backoff_wait: float = MAX_BACKOFF_WAIT, + backoff_factor: float = 0.1, + jitter_ratio: float = 0.1, + respect_retry_after_header: bool = True, + retryable_methods: Iterable[str] = None, + retry_status_codes: Iterable[int] = None + + ) -> None: + self.wrapped_transport = wrapped_transport + if jitter_ratio < 0 or jitter_ratio > 0.5: + raise ValueError(f"jitter ratio should be between 0 and 0.5, actual {jitter_ratio}") + + self.max_attempts = max_attempts + self.backoff_factor = backoff_factor + self.respect_retry_after_header = respect_retry_after_header + self.retryable_methods = frozenset(retryable_methods) if retryable_methods else self.RETRYABLE_METHODS + self.retry_status_codes = frozenset(retry_status_codes) if retry_status_codes else self.RETRYABLE_STATUS_CODES + self.jitter_ratio = jitter_ratio + self.max_backoff_wait = max_backoff_wait + + def _calculate_sleep(self, attempts_made: int, headers: Union[httpx.Headers, Mapping[str, str]]) -> float: + + retry_after_header = (headers.get("Retry-After") or "").strip() + if self.respect_retry_after_header and retry_after_header: + if retry_after_header.isdigit(): + return float(retry_after_header) + + try: + parsed_date = isoparse(retry_after_header).astimezone() # converts to local time + diff = (parsed_date - datetime.now().astimezone()).total_seconds() + if diff > 0: + return min(diff, self.max_backoff_wait) + except ValueError as _ex: + pass + + backoff = self.backoff_factor * (2 ** (attempts_made - 1)) + jitter = (backoff * self.jitter_ratio) * random.choice([1, -1]) + total_backoff = backoff + jitter + return min(total_backoff, self.max_backoff_wait) + + def handle_request(self, request: httpx.Request) -> httpx.Response: + + response = self.wrapped_transport.handle_request(request) + + if request.method not in self.retryable_methods: + return response + + remaining_attempts = self.max_attempts - 1 + attempts_made = 1 + + while True: + + if remaining_attempts < 1 or response.status_code not in self.retry_status_codes: + return response + + response.close() + + sleep_for = self._calculate_sleep(attempts_made, response.headers) + sleep(sleep_for) + + response = self.wrapped_transport.handle_request(request) + + attempts_made += 1 + remaining_attempts -= 1 + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + + response = await self.wrapped_transport.handle_async_request(request) + + if request.method not in self.retryable_methods: + return response + + remaining_attempts = self.max_attempts - 1 + attempts_made = 1 + + while True: + + if remaining_attempts < 1 or response.status_code not in self.retry_status_codes: + return response + + response.close() + + sleep_for = self._calculate_sleep(attempts_made, response.headers) + sleep(sleep_for) + + response = await self.wrapped_transport.handle_async_request(request) + + attempts_made += 1 + remaining_attempts -= 1 \ No newline at end of file diff --git a/src/keboola/http_client/test_async.py b/src/keboola/http_client/test_async.py new file mode 100644 index 0000000..735f14f --- /dev/null +++ b/src/keboola/http_client/test_async.py @@ -0,0 +1,36 @@ +from async_client import AsyncHttpClient +import requests +import asyncio +import time + + +async def get_pokemon(client, url): + resp = await client.get(url) + print(resp.json()['name']) + + +async def async_get(): + pokemons = [] + client = AsyncHttpClient("https://pokeapi.co/api/v2/") + with client: + for n in range(1, 151): + r = await client.get(f"pokemon/{str(n)}") + pokemons.append(r.json()['name']) + + +def normal_get(): + pokemons = [] + baseurl = "https://pokeapi.co/api/v2/pokemon/" + for n in range(1, 151): + url = baseurl + str(n) + r = requests.get(url) + pokemons.append(r.json()['name']) + + +start_time = time.time() +asyncio.run(async_get()) +print(f"Async took {time.time() - start_time} seconds.") + +start_time = time.time() +normal_get() +print(f"Normal took {time.time() - start_time} seconds.") From 1429b685937e08d874259ffc9f9c62fd1a6552cc Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Tue, 7 Mar 2023 16:46:06 +0100 Subject: [PATCH 03/56] methods update, post, patch, delete --- src/keboola/http_client/async_client.py | 272 ++++++++++++++++++----- src/keboola/http_client/retry_wrapper.py | 243 ++++++++++++++------ src/keboola/http_client/test_async.py | 5 - 3 files changed, 392 insertions(+), 128 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 8026c9d..be17872 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -1,10 +1,8 @@ -import functools -import logging import urllib.parse as urlparse from http.cookiejar import CookieJar from typing import Dict, Union, Tuple, Optional -from httpx import AsyncClient, AsyncHTTPTransport, Response +from httpx import AsyncClient, Response from retry_wrapper import RetryTransport @@ -18,7 +16,7 @@ ALLOWED_METHODS = ['GET', 'POST', 'PATCH', 'UPDATE', 'PUT', 'DELETE'] -class AsyncHttpClient(): +class AsyncHttpClient: """ Base class for implementing a simple HTTP client. Typically used as a base for a REST service client. @@ -60,12 +58,12 @@ def __init__(self, base_url: str, max_retries: int = 10, backoff_factor: float = backoff_factor: A back-off factor to apply between attempts. status_forcelist: A set of HTTP status codes that we should force a retry on. e.g. [500,502] default_http_header: Default header to be sent with each request - eg. ```{ + e.g. ```{ 'Content-Type' : 'application/json', 'Accept' : 'application/json' }``` auth_header: Auth header to be sent with each request - eg. `{'Authorization': 'Bearer ' + token}` + e.g. `{'Authorization': 'Bearer ' + token}` auth: Default Authentication tuple or object to attach to (from requests.Session().auth). eg. auth = (user, password) default_params: default parameters to be sent with each request eg. `{'param':'value'}` @@ -84,32 +82,13 @@ def __init__(self, base_url: str, max_retries: int = 10, backoff_factor: float = self._default_params = default_params self.allowed_methods = allowed_methods - self.s = AsyncClient() + self.session = None def _requests_retry_session(self, session=None): - transport = RetryTransport(wrapped_transport=AsyncHTTPTransport(), max_attempts=self.max_retries, - max_backoff_wait=30) - session = session or AsyncClient(transport=transport) - """ - retry = Retry( - total=self.max_retries, - read=self.max_retries, - connect=self.max_retries, - backoff_factor=self.backoff_factor, - status_forcelist=self.status_forcelist, - allowed_methods=self.allowed_methods - ) - adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) - """ - return session - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass + retry_transport = RetryTransport(max_attempts=self.max_retries, + max_backoff_wait=30) + s = session or AsyncClient(transport=retry_transport) + return s def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False): # build URL Specification @@ -124,6 +103,21 @@ def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False return url + def update_auth_header(self, updated_header: Dict, overwrite: bool = False): + """ + Updates the default auth header by providing new values. + + Args: + updated_header: An updated header which will be used to update the current header. + overwrite: If `False`, the existing header will be updated with new header. If `True`, the new header will + overwrite (replace) the current authentication header. + """ + + if overwrite is False: + self._auth_header.update(updated_header) + else: + self._auth_header = updated_header + async def _request_raw(self, method: str, endpoint_path: Optional[str] = None, **kwargs) -> Response: """ Construct a requests call with args and kwargs and process the @@ -144,6 +138,8 @@ async def _request_raw(self, method: str, endpoint_path: Optional[str] = None, * A [`requests.Response`](https://requests.readthedocs.io/en/latest/api/#requests.Response) object. """ + self.session = self.session if self.session else AsyncClient() + # build URL Specification is_absolute_path = kwargs.pop('is_absolute_path', False) url = self._build_url(endpoint_path, is_absolute_path) @@ -159,10 +155,10 @@ async def _request_raw(self, method: str, endpoint_path: Optional[str] = None, * # Auth headers if kwargs.pop('ignore_auth', False) is False: headers.update(self._auth_header) - self.s.headers.update(headers) - self.s.auth = self._auth + self.session.headers.update(headers) + self.session.auth = self._auth - self.s.headers.update(headers) + self.session.headers.update(headers) # Update parameters params = kwargs.pop('params', {}) @@ -177,12 +173,12 @@ async def _request_raw(self, method: str, endpoint_path: Optional[str] = None, * else: kwargs.update({'params': params}) - r = await self._requests_retry_session(session=self.s).request(method, url, **kwargs) + r = await self._requests_retry_session(session=self.session).request(method, url, **kwargs) return r - async def get_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, - is_absolute_path: bool = False, cookies: Cookie = None, - ignore_auth: bool = False, **kwargs) -> Response: + async def get(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, + is_absolute_path: bool = False, cookies: Cookie = None, + ignore_auth: bool = False, **kwargs) -> Response: """ Constructs a requests GET call with specified url and kwargs to process the result. @@ -201,7 +197,7 @@ async def get_raw(self, endpoint_path: Optional[str] = None, params: Dict = None [`urllib.parse.urljoin()`](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin) function. - If set to True, base url will be overridden and the value of the `endpoint_path` will + If set to True, base url will be overridden and the value of the `endpoint_path` will be used instead. cookies: Dict or CookieJar object of cookies to send with the request ignore_auth: Boolean marking, whether the default auth_header should be ignored. @@ -209,19 +205,21 @@ async def get_raw(self, endpoint_path: Optional[str] = None, params: Dict = None [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). Returns: - A [`requests.Response`](https://requests.readthedocs.io/en/latest/api/#requests.Response) object. + A httpx.Response object. + + Raises: + requests.HTTPError: If the API request fails. """ - method = 'GET' - r = await self._request_raw(method, endpoint_path, params=params, headers=headers, cookies=cookies, + r = await self._request_raw(endpoint_path, params=params, headers=headers, cookies=cookies, is_absolute_path=is_absolute_path, ignore_auth=ignore_auth, **kwargs) return r - async def get(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, - is_absolute_path: bool = False, cookies: Cookie = None, - ignore_auth: bool = False, **kwargs) -> Response: + async def post(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, + data: Dict = None, json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, + files: Dict = None, ignore_auth: bool = False, **kwargs) -> Response: """ - Constructs a requests GET call with specified url and kwargs to process the result. + Constructs a requests POST call with specified url and kwargs to process the result. Args: endpoint_path: Relative URL path or absolute URL to which the request will be made. @@ -231,6 +229,8 @@ async def get(self, endpoint_path: Optional[str] = None, params: Dict = None, he or used as an absolute URL. params: Dictionary to send in the query string for the request. headers: Dictionary of HTTP Headers to send with the request. + data: Dictionary to send in the body of the request. + json: A JSON serializable Python object to send in the body of the request. is_absolute_path: A boolean value specifying, whether the URL specified in `endpoint_path` parameter is an absolute path or not. @@ -239,7 +239,7 @@ async def get(self, endpoint_path: Optional[str] = None, params: Dict = None, he function. If set to True, base url will be overridden and the value of the `endpoint_path` will - used instead. + be used instead. cookies: Dict or CookieJar object of cookies to send with the request files: Dictionary of 'name': file-like-objects (or {'name': file-tuple}) for multipart encoding upload. ignore_auth: Boolean marking, whether the default auth_header should be ignored. @@ -247,12 +247,182 @@ async def get(self, endpoint_path: Optional[str] = None, params: Dict = None, he [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). Returns: - A JSON-encoded response of the request. + A httpx.Response object. + """ - Raises: - requests.HTTPError: If the API request fails. + method = 'POST' + r = await self._request_raw(method, endpoint_path, params=params, headers=headers, data=data, json=json, + cookies=cookies, is_absolute_path=is_absolute_path, files=files, + ignore_auth=ignore_auth, **kwargs) + + return r + + async def patch(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, + data: Dict = None, json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, + files: Dict = None, ignore_auth: bool = False, **kwargs) -> Response: """ + Constructs a requests PATCH call with specified url and kwargs to process the result. + + Args: + endpoint_path: Relative URL path or absolute URL to which the request will be made. + By default a relative path is expected and will be appended to the `self.base_url` value. - r = await self.get_raw(endpoint_path, params=params, headers=headers, cookies=cookies, - is_absolute_path=is_absolute_path, ignore_auth=ignore_auth, **kwargs) + Depending on the value of `is_absolute_path`, the value will be either appended to `self.base_url`, + or used as an absolute URL. + params: Dictionary to send in the query string for the request. + headers: Dictionary of HTTP Headers to send with the request. + data: Dictionary to send in the body of the request. + json: A JSON serializable Python object to send in the body of the request. + is_absolute_path: A boolean value specifying, whether the URL specified in `endpoint_path` parameter + is an absolute path or not. + + If set to False, the value of `endpoint_path` will be appended to `self.base_url` using + [`urllib.parse.urljoin()`](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin) + function. + + If set to True, base url will be overridden and the value of the `endpoint_path` will + be used instead. + cookies: Dict or CookieJar object of cookies to send with the request + files: Dictionary of 'name': file-like-objects (or {'name': file-tuple}) for multipart encoding upload. + ignore_auth: Boolean marking, whether the default auth_header should be ignored. + **kwargs: All other keyword arguments supported by + [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). + + Returns: + A httpx.Response object. + """ + + method = 'PATCH' + r = await self._request_raw(method, endpoint_path, params=params, headers=headers, data=data, json=json, + cookies=cookies, is_absolute_path=is_absolute_path, files=files, + ignore_auth=ignore_auth, **kwargs) return r + + async def update(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, + data: Dict = None, json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, + files: Dict = None, ignore_auth: bool = False, **kwargs) -> Response: + """ + Constructs a requests UPDATE call with specified url and kwargs to process the result. + + Args: + endpoint_path: Relative URL path or absolute URL to which the request will be made. + By default a relative path is expected and will be appended to the `self.base_url` value. + + Depending on the value of `is_absolute_path`, the value will be either appended to `self.base_url`, + or used as an absolute URL. + params: Dictionary to send in the query string for the request. + headers: Dictionary of HTTP Headers to send with the request. + data: Dictionary to send in the body of the request. + json: A JSON serializable Python object to send in the body of the request. + is_absolute_path: A boolean value specifying, whether the URL specified in `endpoint_path` parameter + is an absolute path or not. + + If set to False, the value of `endpoint_path` will be appended to `self.base_url` using + [`urllib.parse.urljoin()`](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin) + function. + + If set to True, base url will be overridden and the value of the `endpoint_path` will + be used instead. + cookies: Dict or CookieJar object of cookies to send with the request + files: Dictionary of 'name': file-like-objects (or {'name': file-tuple}) for multipart encoding upload. + ignore_auth: Boolean marking, whether the default auth_header should be ignored. + **kwargs: All other keyword arguments supported by + [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). + + Returns: + A httpx.Response object. + """ + + method = 'UPDATE' + r = await self._request_raw(method, endpoint_path, params=params, headers=headers, data=data, json=json, + cookies=cookies, is_absolute_path=is_absolute_path, files=files, + ignore_auth=ignore_auth, **kwargs) + return r + + async def put(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, + data: Dict = None, json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, + files: Dict = None, ignore_auth: bool = False, **kwargs) -> Response: + """ + Constructs a requests PUT call with specified url and kwargs to process the result. + + Args: + endpoint_path: Relative URL path or absolute URL to which the request will be made. + By default a relative path is expected and will be appended to the `self.base_url` value. + + Depending on the value of `is_absolute_path`, the value will be either appended to `self.base_url`, + or used as an absolute URL. + params: Dictionary to send in the query string for the request. + headers: Dictionary of HTTP Headers to send with the request. + data: Dictionary to send in the body of the request. + json: A JSON serializable Python object to send in the body of the request. + is_absolute_path: A boolean value specifying, whether the URL specified in `endpoint_path` parameter + is an absolute path or not. + + If set to False, the value of `endpoint_path` will be appended to `self.base_url` using + [`urllib.parse.urljoin()`](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin) + function. + + If set to True, base url will be overridden and the value of the `endpoint_path` will + be used instead. + cookies: Dict or CookieJar object of cookies to send with the request + files: Dictionary of 'name': file-like-objects (or {'name': file-tuple}) for multipart encoding upload. + ignore_auth: Boolean marking, whether the default auth_header should be ignored. + **kwargs: All other keyword arguments supported by + [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). + + Returns: + A httpx.Response object. + """ + + method = 'PUT' + r = await self._request_raw(method, endpoint_path, params=params, headers=headers, data=data, json=json, + cookies=cookies, is_absolute_path=is_absolute_path, files=files, + ignore_auth=ignore_auth, **kwargs) + return r + + async def delete_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, + data: Dict = None, json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, + files: Dict = None, ignore_auth: bool = False, **kwargs) -> Response: + """ + Constructs a requests DELETE call with specified url and kwargs to process the result. + + Args: + endpoint_path: Relative URL path or absolute URL to which the request will be made. + By default a relative path is expected and will be appended to the `self.base_url` value. + + Depending on the value of `is_absolute_path`, the value will be either appended to `self.base_url`, + or used as an absolute URL. + params: Dictionary to send in the query string for the request. + headers: Dictionary of HTTP Headers to send with the request. + data: Dictionary to send in the body of the request. + json: A JSON serializable Python object to send in the body of the request. + is_absolute_path: A boolean value specifying, whether the URL specified in `endpoint_path` parameter + is an absolute path or not. + + If set to False, the value of `endpoint_path` will be appended to `self.base_url` using + [`urllib.parse.urljoin()`](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin) + function. + + If set to True, base url will be overridden and the value of the `endpoint_path` will + be used instead. + cookies: Dict or CookieJar object of cookies to send with the request + files: Dictionary of 'name': file-like-objects (or {'name': file-tuple}) for multipart encoding upload. + ignore_auth: Boolean marking, whether the default auth_header should be ignored. + **kwargs: All other keyword arguments supported by + [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). + + Returns: + A httpx.Response object. + """ + + method = 'DELETE' + r = await self._request_raw(method, endpoint_path, params=params, headers=headers, data=data, json=json, + cookies=cookies, is_absolute_path=is_absolute_path, files=files, + ignore_auth=ignore_auth, **kwargs) + return r + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass diff --git a/src/keboola/http_client/retry_wrapper.py b/src/keboola/http_client/retry_wrapper.py index bc19900..3e60a13 100644 --- a/src/keboola/http_client/retry_wrapper.py +++ b/src/keboola/http_client/retry_wrapper.py @@ -1,115 +1,214 @@ -from datetime import datetime +import asyncio import random -from time import sleep +import time +from datetime import datetime +from functools import partial +from http import HTTPStatus +from typing import Any, Callable, Coroutine, Iterable, Mapping, Union, Type import httpx -from typing import Union, Iterable, Mapping - from dateutil.parser import isoparse -class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport): +class RetryTransport(httpx.AsyncBaseTransport): """ - Credits to: https://github.com/matt-mercer + Credits to: https://github.com/DeepakArora76 + A custom HTTP transport that automatically retries requests using an exponential backoff strategy + for specific HTTP status codes and request methods. + + Args: + max_attempts (int, optional): The maximum number of times to retry a request before giving up. Defaults to 10. + max_backoff_wait (float, optional): The maximum time to wait between retries in seconds. Defaults to 60. + backoff_factor (float, optional): The factor by which the wait time increases with each retry attempt. + Defaults to 0.1. + jitter_ratio (float, optional): The amount of jitter to add to the backoff time. Jitter is a random + value added to the backoff time to avoid a "thundering herd" effect. The value should be between 0 and 0.5. + Defaults to 0.1. + respect_retry_after_header (bool, optional): Whether to respect the Retry-After header in HTTP responses + when deciding how long to wait before retrying. Defaults to True. + retryable_methods (Iterable[str], optional): The HTTP methods that can be retried. Defaults to + ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]. + retry_status_codes (Iterable[int], optional): The HTTP status codes that can be retried. Defaults to + [429, 502, 503, 504]. + + Attributes: + _wrapped_transport (Union[httpx.BaseTransport, httpx.AsyncBaseTransport]): The underlying HTTP transport + being wrapped. + _max_attempts (int): The maximum number of times to retry a request. + _backoff_factor (float): The factor by which the wait time increases with each retry attempt. + _respect_retry_after_header (bool): Whether to respect the Retry-After header in HTTP responses. + _retryable_methods (frozenset): The HTTP methods that can be retried. + _retry_status_codes (frozenset): The HTTP status codes that can be retried. + _jitter_ratio (float): The amount of jitter to add to the backoff time. + _max_backoff_wait (float): The maximum time to wait between retries in seconds. + """ - RETRYABLE_METHODS = frozenset( - ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"] + RETRYABLE_METHODS = frozenset(["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]) + RETRYABLE_STATUS_CODES = frozenset( + [ + HTTPStatus.TOO_MANY_REQUESTS, + HTTPStatus.BAD_GATEWAY, + HTTPStatus.SERVICE_UNAVAILABLE, + HTTPStatus.GATEWAY_TIMEOUT, + ] ) - RETRYABLE_STATUS_CODES = frozenset([413, 429, 503, 504]) - MAX_BACKOFF_WAIT = 60 def __init__( self, - wrapped_transport: Union[httpx.BaseTransport, httpx.AsyncBaseTransport], max_attempts: int = 10, max_backoff_wait: float = MAX_BACKOFF_WAIT, backoff_factor: float = 0.1, jitter_ratio: float = 0.1, respect_retry_after_header: bool = True, retryable_methods: Iterable[str] = None, - retry_status_codes: Iterable[int] = None - + retry_status_codes: Iterable[int] = None, ) -> None: - self.wrapped_transport = wrapped_transport + """ + Initializes the instance of RetryTransport class with the given parameters. + + Args: + max_attempts (int, optional): + The maximum number of times the request can be retried in case of failure. + Defaults to 10. + max_backoff_wait (float, optional): + The maximum amount of time (in seconds) to wait before retrying a request. + Defaults to 60. + backoff_factor (float, optional): + The factor by which the waiting time will be multiplied in each retry attempt. + Defaults to 0.1. + jitter_ratio (float, optional): + The ratio of randomness added to the waiting time to prevent simultaneous retries. + Should be between 0 and 0.5. Defaults to 0.1. + respect_retry_after_header (bool, optional): + A flag to indicate if the Retry-After header should be respected. + If True, the waiting time specified in Retry-After header is used for the waiting time. + Defaults to True. + retryable_methods (Iterable[str], optional): + The HTTP methods that can be retried. Defaults to ['HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE']. + retry_status_codes (Iterable[int], optional): + The HTTP status codes that can be retried. + Defaults to [429, 502, 503, 504]. + """ + self._wrapped_transport = httpx.AsyncBaseTransport if jitter_ratio < 0 or jitter_ratio > 0.5: - raise ValueError(f"jitter ratio should be between 0 and 0.5, actual {jitter_ratio}") - - self.max_attempts = max_attempts - self.backoff_factor = backoff_factor - self.respect_retry_after_header = respect_retry_after_header - self.retryable_methods = frozenset(retryable_methods) if retryable_methods else self.RETRYABLE_METHODS - self.retry_status_codes = frozenset(retry_status_codes) if retry_status_codes else self.RETRYABLE_STATUS_CODES - self.jitter_ratio = jitter_ratio - self.max_backoff_wait = max_backoff_wait - - def _calculate_sleep(self, attempts_made: int, headers: Union[httpx.Headers, Mapping[str, str]]) -> float: + raise ValueError( + f"Jitter ratio should be between 0 and 0.5, actual {jitter_ratio}" + ) + + self._max_attempts = max_attempts + self._backoff_factor = backoff_factor + self._respect_retry_after_header = respect_retry_after_header + self._retryable_methods = ( + frozenset(retryable_methods) + if retryable_methods + else self.RETRYABLE_METHODS + ) + self._retry_status_codes = ( + frozenset(retry_status_codes) + if retry_status_codes + else self.RETRYABLE_STATUS_CODES + ) + self._jitter_ratio = jitter_ratio + self._max_backoff_wait = max_backoff_wait + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + """Sends an HTTP request, possibly with retries. + + Args: + request: The request to perform. + + Returns: + The response. + + """ + transport: Type[httpx.AsyncBaseTransport] = self._wrapped_transport + if request.method in self._retryable_methods: + send_method = partial(transport.handle_async_request) + response = await self._retry_operation_async(request, send_method) + else: + response = await transport.handle_async_request(request) + return response + + async def aclose(self) -> None: + """ + Closes the underlying HTTP transport, terminating all outstanding connections and rejecting any further + requests. + + This should be called before the object is dereferenced, to ensure that connections are properly cleaned up. + """ + transport: Type[httpx.AsyncBaseTransport] = self._wrapped_transport + await transport.aclose(self) + + def _calculate_sleep( + self, attempts_made: int, headers: Union[httpx.Headers, Mapping[str, str]] + ) -> float: + # Retry-After + # The Retry-After response HTTP header indicates how long the user agent should wait before + # making a follow-up request. There are three main cases this header is used: + # - When sent with a 503 (Service Unavailable) response, this indicates how long the service + # is expected to be unavailable. + # - When sent with a 429 (Too Many Requests) response, this indicates how long to wait before + # making a new request. + # - When sent with a redirect response, such as 301 (Moved Permanently), this indicates the + # minimum time that the user agent is asked to wait before issuing the redirected request. retry_after_header = (headers.get("Retry-After") or "").strip() - if self.respect_retry_after_header and retry_after_header: + if self._respect_retry_after_header and retry_after_header: if retry_after_header.isdigit(): return float(retry_after_header) try: - parsed_date = isoparse(retry_after_header).astimezone() # converts to local time + parsed_date = isoparse( + retry_after_header + ).astimezone() # converts to local time diff = (parsed_date - datetime.now().astimezone()).total_seconds() if diff > 0: - return min(diff, self.max_backoff_wait) - except ValueError as _ex: + return min(diff, self._max_backoff_wait) + except ValueError: pass - backoff = self.backoff_factor * (2 ** (attempts_made - 1)) - jitter = (backoff * self.jitter_ratio) * random.choice([1, -1]) + backoff = self._backoff_factor * (2 ** (attempts_made - 1)) + jitter = (backoff * self._jitter_ratio) * random.choice([1, -1]) total_backoff = backoff + jitter - return min(total_backoff, self.max_backoff_wait) - - def handle_request(self, request: httpx.Request) -> httpx.Response: - - response = self.wrapped_transport.handle_request(request) - - if request.method not in self.retryable_methods: - return response - - remaining_attempts = self.max_attempts - 1 - attempts_made = 1 + return min(total_backoff, self._max_backoff_wait) + async def _retry_operation_async( + self, + request: httpx.Request, + send_method: Callable[..., Coroutine[Any, Any, httpx.Response]], + ) -> httpx.Response: + remaining_attempts = self._max_attempts + attempts_made = 0 while True: - - if remaining_attempts < 1 or response.status_code not in self.retry_status_codes: + if attempts_made > 0: + await asyncio.sleep(self._calculate_sleep(attempts_made, {})) + response = await send_method(request) + if ( + remaining_attempts < 1 + or response.status_code not in self._retry_status_codes + ): return response - - response.close() - - sleep_for = self._calculate_sleep(attempts_made, response.headers) - sleep(sleep_for) - - response = self.wrapped_transport.handle_request(request) - + await response.aclose() attempts_made += 1 remaining_attempts -= 1 - async def handle_async_request(self, request: httpx.Request) -> httpx.Response: - - response = await self.wrapped_transport.handle_async_request(request) - - if request.method not in self.retryable_methods: - return response - - remaining_attempts = self.max_attempts - 1 - attempts_made = 1 - + def _retry_operation( + self, + request: httpx.Request, + send_method: Callable[..., httpx.Response], + ) -> httpx.Response: + remaining_attempts = self._max_attempts + attempts_made = 0 while True: - - if remaining_attempts < 1 or response.status_code not in self.retry_status_codes: + if attempts_made > 0: + time.sleep(self._calculate_sleep(attempts_made, {})) + response = send_method(request) + if ( + remaining_attempts < 1 + or response.status_code not in self._retry_status_codes + ): return response - response.close() - - sleep_for = self._calculate_sleep(attempts_made, response.headers) - sleep(sleep_for) - - response = await self.wrapped_transport.handle_async_request(request) - attempts_made += 1 - remaining_attempts -= 1 \ No newline at end of file + remaining_attempts -= 1 diff --git a/src/keboola/http_client/test_async.py b/src/keboola/http_client/test_async.py index 735f14f..d7ffc44 100644 --- a/src/keboola/http_client/test_async.py +++ b/src/keboola/http_client/test_async.py @@ -4,11 +4,6 @@ import time -async def get_pokemon(client, url): - resp = await client.get(url) - print(resp.json()['name']) - - async def async_get(): pokemons = [] client = AsyncHttpClient("https://pokeapi.co/api/v2/") From 162833d21ea0b40b41de218df0c1d5f10a29b693 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Thu, 16 Mar 2023 11:42:20 +0100 Subject: [PATCH 04/56] update methods --- src/keboola/http_client/async_client.py | 4 ++-- src/keboola/http_client/retry_wrapper.py | 8 ++++---- src/keboola/http_client/test_async.py | 21 +++++++++++---------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index be17872..a3096c7 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -210,8 +210,8 @@ async def get(self, endpoint_path: Optional[str] = None, params: Dict = None, he Raises: requests.HTTPError: If the API request fails. """ - - r = await self._request_raw(endpoint_path, params=params, headers=headers, cookies=cookies, + method = 'GET' + r = await self._request_raw(method, endpoint_path, params=params, headers=headers, cookies=cookies, is_absolute_path=is_absolute_path, ignore_auth=ignore_auth, **kwargs) return r diff --git a/src/keboola/http_client/retry_wrapper.py b/src/keboola/http_client/retry_wrapper.py index 3e60a13..1bfe74c 100644 --- a/src/keboola/http_client/retry_wrapper.py +++ b/src/keboola/http_client/retry_wrapper.py @@ -27,12 +27,12 @@ class RetryTransport(httpx.AsyncBaseTransport): respect_retry_after_header (bool, optional): Whether to respect the Retry-After header in HTTP responses when deciding how long to wait before retrying. Defaults to True. retryable_methods (Iterable[str], optional): The HTTP methods that can be retried. Defaults to - ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]. + ["GET", "PUT", "DELETE", "PATCH"]. retry_status_codes (Iterable[int], optional): The HTTP status codes that can be retried. Defaults to [429, 502, 503, 504]. Attributes: - _wrapped_transport (Union[httpx.BaseTransport, httpx.AsyncBaseTransport]): The underlying HTTP transport + _wrapped_transport (httpx.AsyncBaseTransport): The underlying HTTP transport being wrapped. _max_attempts (int): The maximum number of times to retry a request. _backoff_factor (float): The factor by which the wait time increases with each retry attempt. @@ -44,7 +44,7 @@ class RetryTransport(httpx.AsyncBaseTransport): """ - RETRYABLE_METHODS = frozenset(["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]) + RETRYABLE_METHODS = frozenset(["GET", "PUT", "DELETE", "PATCH"]) RETRYABLE_STATUS_CODES = frozenset( [ HTTPStatus.TOO_MANY_REQUESTS, @@ -86,7 +86,7 @@ def __init__( If True, the waiting time specified in Retry-After header is used for the waiting time. Defaults to True. retryable_methods (Iterable[str], optional): - The HTTP methods that can be retried. Defaults to ['HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE']. + The HTTP methods that can be retried. Defaults to ["GET", "PUT", "DELETE", "PATCH"]. retry_status_codes (Iterable[int], optional): The HTTP status codes that can be retried. Defaults to [429, 502, 503, 504]. diff --git a/src/keboola/http_client/test_async.py b/src/keboola/http_client/test_async.py index d7ffc44..87c0556 100644 --- a/src/keboola/http_client/test_async.py +++ b/src/keboola/http_client/test_async.py @@ -3,29 +3,30 @@ import asyncio import time +common_url = "https://pokeapi.co/api/v2/" +nr_of_pokemons = 202 -async def async_get(): + +async def async_get(url, nr_of_pokemons): pokemons = [] - client = AsyncHttpClient("https://pokeapi.co/api/v2/") + client = AsyncHttpClient(url) with client: - for n in range(1, 151): + for n in range(1, nr_of_pokemons): r = await client.get(f"pokemon/{str(n)}") pokemons.append(r.json()['name']) -def normal_get(): +def normal_get(url, nr_of_pokemons): pokemons = [] - baseurl = "https://pokeapi.co/api/v2/pokemon/" - for n in range(1, 151): - url = baseurl + str(n) - r = requests.get(url) + for n in range(1, nr_of_pokemons): + r = requests.get(url + f"pokemon/{str(n)}") pokemons.append(r.json()['name']) start_time = time.time() -asyncio.run(async_get()) +asyncio.run(async_get(common_url, nr_of_pokemons)) print(f"Async took {time.time() - start_time} seconds.") start_time = time.time() -normal_get() +normal_get(common_url, nr_of_pokemons) print(f"Normal took {time.time() - start_time} seconds.") From 8d4a9ac76470698823401faa0b409df34e5d2cc4 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Fri, 21 Apr 2023 15:44:43 +0200 Subject: [PATCH 05/56] httpx updates --- requirements.txt | 3 +- src/keboola/http_client/async_client.py | 518 ++++-------------- .../http_client/example_async_poekapi.py | 50 ++ .../http_client/example_async_trivia.py | 27 + src/keboola/http_client/test_async.py | 32 -- tests/test_async.py | 82 +++ 6 files changed, 261 insertions(+), 451 deletions(-) create mode 100644 src/keboola/http_client/example_async_poekapi.py create mode 100644 src/keboola/http_client/example_async_trivia.py delete mode 100644 src/keboola/http_client/test_async.py create mode 100644 tests/test_async.py diff --git a/requirements.txt b/requirements.txt index e6954cd..7ffd6c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests -httpx \ No newline at end of file +httpx==0.20.0 +async_lru \ No newline at end of file diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index a3096c7..34e612c 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -1,428 +1,110 @@ -import urllib.parse as urlparse -from http.cookiejar import CookieJar -from typing import Dict, Union, Tuple, Optional - -from httpx import AsyncClient, Response - -from retry_wrapper import RetryTransport - -# import requests -# from requests.adapters import HTTPAdapter -# from requests.packages.urllib3.util.retry import Retry # noqa - -Cookie = Union[Dict[str, str], CookieJar] - -METHOD_RETRY_WHITELIST = ('GET', 'POST', 'PATCH', 'UPDATE', 'PUT', 'DELETE') -ALLOWED_METHODS = ['GET', 'POST', 'PATCH', 'UPDATE', 'PUT', 'DELETE'] +import httpx +import asyncio +from typing import Optional, Dict, Any, List +from urllib.parse import urljoin class AsyncHttpClient: """ - Base class for implementing a simple HTTP client. Typically used as a base for a REST service client. - - - Usage: - - ```python - from keboola.http_client import HttpClient - - BASE_URL = 'https://connection.keboola.com/v2/storage/' - AUTH_HEADER = { - 'x-storageapi-token': '1234-STORAGETOKENSTRING' - } - DEFAULT_PARAMS = { - 'include': 'columns' - } - DEFAULT_HEADER = { - 'Content-Type': 'application/json' - } - - cl = HttpClient(BASE_URL, default_http_header=DEFAULT_HEADER, - auth_header=AUTH_HEADER, default_params=DEFAULT_PARAMS) - - files_response = cl.get("files", params={"showExpired": True}) - ``` - + An asynchronous HTTP client that simplifies making requests to a specific API. """ - def __init__(self, base_url: str, max_retries: int = 10, backoff_factor: float = 0.3, - status_forcelist: Tuple[int, ...] = (500, 502, 504), default_http_header: Dict = None, - auth_header: Dict = None, auth: Tuple = None, default_params: Dict = None, - allowed_methods: Tuple = METHOD_RETRY_WHITELIST): - """ - Create an endpoint. - - Args: - base_url: The base URL for this endpoint. e.g. https://exampleservice.com/api_v1/ - max_retries: Total number of retries to allow. - backoff_factor: A back-off factor to apply between attempts. - status_forcelist: A set of HTTP status codes that we should force a retry on. e.g. [500,502] - default_http_header: Default header to be sent with each request - e.g. ```{ - 'Content-Type' : 'application/json', - 'Accept' : 'application/json' - }``` - auth_header: Auth header to be sent with each request - e.g. `{'Authorization': 'Bearer ' + token}` - auth: Default Authentication tuple or object to attach to (from requests.Session().auth). - eg. auth = (user, password) - default_params: default parameters to be sent with each request eg. `{'param':'value'}` - allowed_methods (tuple): Set of upper-cased HTTP method verbs that we should retry on. - """ - if base_url is None: - raise ValueError("Base URL is required.") - # Add trailing slash because of nature of urllib.parse.urljoin() - self.base_url = base_url if base_url.endswith('/') else base_url + '/' - self.max_retries = max_retries - self.backoff_factor = backoff_factor - self.status_forcelist = status_forcelist - self._auth = auth - self._auth_header = auth_header if auth_header else {} - self._default_header = default_http_header if default_http_header else {} - self._default_params = default_params - self.allowed_methods = allowed_methods - - self.session = None - - def _requests_retry_session(self, session=None): - retry_transport = RetryTransport(max_attempts=self.max_retries, - max_backoff_wait=30) - s = session or AsyncClient(transport=retry_transport) - return s - - def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False): - # build URL Specification - url_path = str(endpoint_path).strip() if endpoint_path is not None else '' - - if not url_path: - url = self.base_url - elif not is_absolute_path: - url = urlparse.urljoin(str(self.base_url), endpoint_path) - else: - url = endpoint_path - - return url - - def update_auth_header(self, updated_header: Dict, overwrite: bool = False): - """ - Updates the default auth header by providing new values. - - Args: - updated_header: An updated header which will be used to update the current header. - overwrite: If `False`, the existing header will be updated with new header. If `True`, the new header will - overwrite (replace) the current authentication header. - """ - - if overwrite is False: - self._auth_header.update(updated_header) - else: - self._auth_header = updated_header - - async def _request_raw(self, method: str, endpoint_path: Optional[str] = None, **kwargs) -> Response: - """ - Construct a requests call with args and kwargs and process the - results. - - Args: - method: A HTTP method to be used. One of PUT/POST/PATCH/GET/UPDATE/DELETE - endpoint_path (Optional[str]): Optional full URL or a relative URL path. If empty the base_url is used. - **kwargs: Key word arguments to pass to the - [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). - Accepts supported params in requests.sessions.Session#request - eg. params = {'locId':'1'}, header = {some additional header} - parameters and headers are appended to the default ones - ignore_auth - True to skip authentication - is_absolute_path - False to append URL to base url; True to override base url with value of url arg. - - Returns: - A [`requests.Response`](https://requests.readthedocs.io/en/latest/api/#requests.Response) object. - """ - - self.session = self.session if self.session else AsyncClient() - - # build URL Specification - is_absolute_path = kwargs.pop('is_absolute_path', False) - url = self._build_url(endpoint_path, is_absolute_path) - - # Update headers - headers = kwargs.pop('headers', {}) - if headers is None: - headers = {} - - # Default headers - headers.update(self._default_header) - - # Auth headers - if kwargs.pop('ignore_auth', False) is False: - headers.update(self._auth_header) - self.session.headers.update(headers) - self.session.auth = self._auth - - self.session.headers.update(headers) - - # Update parameters - params = kwargs.pop('params', {}) - if params is None: - params = {} - - # Default parameters - if self._default_params is not None: - all_pars = {**params, **self._default_params} - kwargs.update({'params': all_pars}) - - else: - kwargs.update({'params': params}) - - r = await self._requests_retry_session(session=self.session).request(method, url, **kwargs) - return r - - async def get(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, - is_absolute_path: bool = False, cookies: Cookie = None, - ignore_auth: bool = False, **kwargs) -> Response: - """ - Constructs a requests GET call with specified url and kwargs to process the result. + def __init__( + self, + base_url: str, + retries: int = 3, + timeout: Optional[float] = None, + verify_ssl: bool = True, + retry_status_codes: Optional[List[int]] = None, + rate_limit: Optional[int] = None, + default_params: Optional[Dict[str, str]] = None, + auth_header: Optional[Dict[str, str]] = None, + user_agent: Optional[str] = None, + ): + """ + Initialize the AsyncHttpClientV2 instance. Args: - endpoint_path: Relative URL path or absolute URL to which the request will be made. - By default a relative path is expected and will be appended to the `self.base_url` value. - - Depending on the value of `is_absolute_path`, the value will be either appended to `self.base_url`, - or used as an absolute URL. - params: Dictionary to send in the query string for the request. - headers: Dictionary of HTTP Headers to send with the request. - is_absolute_path: A boolean value specifying, whether the URL specified in `endpoint_path` parameter - is an absolute path or not. - - If set to False, the value of `endpoint_path` will be appended to `self.base_url` using - [`urllib.parse.urljoin()`](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin) - function. - - If set to True, base url will be overridden and the value of the `endpoint_path` will be - used instead. - cookies: Dict or CookieJar object of cookies to send with the request - ignore_auth: Boolean marking, whether the default auth_header should be ignored. - **kwargs: All other keyword arguments supported by - [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). - - Returns: - A httpx.Response object. - - Raises: - requests.HTTPError: If the API request fails. - """ - method = 'GET' - r = await self._request_raw(method, endpoint_path, params=params, headers=headers, cookies=cookies, - is_absolute_path=is_absolute_path, ignore_auth=ignore_auth, **kwargs) - return r - - async def post(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, - data: Dict = None, json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, - files: Dict = None, ignore_auth: bool = False, **kwargs) -> Response: - """ - Constructs a requests POST call with specified url and kwargs to process the result. - - Args: - endpoint_path: Relative URL path or absolute URL to which the request will be made. - By default a relative path is expected and will be appended to the `self.base_url` value. - - Depending on the value of `is_absolute_path`, the value will be either appended to `self.base_url`, - or used as an absolute URL. - params: Dictionary to send in the query string for the request. - headers: Dictionary of HTTP Headers to send with the request. - data: Dictionary to send in the body of the request. - json: A JSON serializable Python object to send in the body of the request. - is_absolute_path: A boolean value specifying, whether the URL specified in `endpoint_path` parameter - is an absolute path or not. - - If set to False, the value of `endpoint_path` will be appended to `self.base_url` using - [`urllib.parse.urljoin()`](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin) - function. - - If set to True, base url will be overridden and the value of the `endpoint_path` will - be used instead. - cookies: Dict or CookieJar object of cookies to send with the request - files: Dictionary of 'name': file-like-objects (or {'name': file-tuple}) for multipart encoding upload. - ignore_auth: Boolean marking, whether the default auth_header should be ignored. - **kwargs: All other keyword arguments supported by - [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). - - Returns: - A httpx.Response object. - """ - - method = 'POST' - r = await self._request_raw(method, endpoint_path, params=params, headers=headers, data=data, json=json, - cookies=cookies, is_absolute_path=is_absolute_path, files=files, - ignore_auth=ignore_auth, **kwargs) - - return r - - async def patch(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, - data: Dict = None, json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, - files: Dict = None, ignore_auth: bool = False, **kwargs) -> Response: - """ - Constructs a requests PATCH call with specified url and kwargs to process the result. - - Args: - endpoint_path: Relative URL path or absolute URL to which the request will be made. - By default a relative path is expected and will be appended to the `self.base_url` value. - - Depending on the value of `is_absolute_path`, the value will be either appended to `self.base_url`, - or used as an absolute URL. - params: Dictionary to send in the query string for the request. - headers: Dictionary of HTTP Headers to send with the request. - data: Dictionary to send in the body of the request. - json: A JSON serializable Python object to send in the body of the request. - is_absolute_path: A boolean value specifying, whether the URL specified in `endpoint_path` parameter - is an absolute path or not. - - If set to False, the value of `endpoint_path` will be appended to `self.base_url` using - [`urllib.parse.urljoin()`](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin) - function. - - If set to True, base url will be overridden and the value of the `endpoint_path` will - be used instead. - cookies: Dict or CookieJar object of cookies to send with the request - files: Dictionary of 'name': file-like-objects (or {'name': file-tuple}) for multipart encoding upload. - ignore_auth: Boolean marking, whether the default auth_header should be ignored. - **kwargs: All other keyword arguments supported by - [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). - - Returns: - A httpx.Response object. - """ - - method = 'PATCH' - r = await self._request_raw(method, endpoint_path, params=params, headers=headers, data=data, json=json, - cookies=cookies, is_absolute_path=is_absolute_path, files=files, - ignore_auth=ignore_auth, **kwargs) - return r - - async def update(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, - data: Dict = None, json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, - files: Dict = None, ignore_auth: bool = False, **kwargs) -> Response: - """ - Constructs a requests UPDATE call with specified url and kwargs to process the result. - - Args: - endpoint_path: Relative URL path or absolute URL to which the request will be made. - By default a relative path is expected and will be appended to the `self.base_url` value. - - Depending on the value of `is_absolute_path`, the value will be either appended to `self.base_url`, - or used as an absolute URL. - params: Dictionary to send in the query string for the request. - headers: Dictionary of HTTP Headers to send with the request. - data: Dictionary to send in the body of the request. - json: A JSON serializable Python object to send in the body of the request. - is_absolute_path: A boolean value specifying, whether the URL specified in `endpoint_path` parameter - is an absolute path or not. - - If set to False, the value of `endpoint_path` will be appended to `self.base_url` using - [`urllib.parse.urljoin()`](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin) - function. - - If set to True, base url will be overridden and the value of the `endpoint_path` will - be used instead. - cookies: Dict or CookieJar object of cookies to send with the request - files: Dictionary of 'name': file-like-objects (or {'name': file-tuple}) for multipart encoding upload. - ignore_auth: Boolean marking, whether the default auth_header should be ignored. - **kwargs: All other keyword arguments supported by - [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). - - Returns: - A httpx.Response object. - """ - - method = 'UPDATE' - r = await self._request_raw(method, endpoint_path, params=params, headers=headers, data=data, json=json, - cookies=cookies, is_absolute_path=is_absolute_path, files=files, - ignore_auth=ignore_auth, **kwargs) - return r - - async def put(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, - data: Dict = None, json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, - files: Dict = None, ignore_auth: bool = False, **kwargs) -> Response: - """ - Constructs a requests PUT call with specified url and kwargs to process the result. - - Args: - endpoint_path: Relative URL path or absolute URL to which the request will be made. - By default a relative path is expected and will be appended to the `self.base_url` value. - - Depending on the value of `is_absolute_path`, the value will be either appended to `self.base_url`, - or used as an absolute URL. - params: Dictionary to send in the query string for the request. - headers: Dictionary of HTTP Headers to send with the request. - data: Dictionary to send in the body of the request. - json: A JSON serializable Python object to send in the body of the request. - is_absolute_path: A boolean value specifying, whether the URL specified in `endpoint_path` parameter - is an absolute path or not. - - If set to False, the value of `endpoint_path` will be appended to `self.base_url` using - [`urllib.parse.urljoin()`](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin) - function. - - If set to True, base url will be overridden and the value of the `endpoint_path` will - be used instead. - cookies: Dict or CookieJar object of cookies to send with the request - files: Dictionary of 'name': file-like-objects (or {'name': file-tuple}) for multipart encoding upload. - ignore_auth: Boolean marking, whether the default auth_header should be ignored. - **kwargs: All other keyword arguments supported by - [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). - - Returns: - A httpx.Response object. - """ - - method = 'PUT' - r = await self._request_raw(method, endpoint_path, params=params, headers=headers, data=data, json=json, - cookies=cookies, is_absolute_path=is_absolute_path, files=files, - ignore_auth=ignore_auth, **kwargs) - return r - - async def delete_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, - data: Dict = None, json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, - files: Dict = None, ignore_auth: bool = False, **kwargs) -> Response: - """ - Constructs a requests DELETE call with specified url and kwargs to process the result. - - Args: - endpoint_path: Relative URL path or absolute URL to which the request will be made. - By default a relative path is expected and will be appended to the `self.base_url` value. - - Depending on the value of `is_absolute_path`, the value will be either appended to `self.base_url`, - or used as an absolute URL. - params: Dictionary to send in the query string for the request. - headers: Dictionary of HTTP Headers to send with the request. - data: Dictionary to send in the body of the request. - json: A JSON serializable Python object to send in the body of the request. - is_absolute_path: A boolean value specifying, whether the URL specified in `endpoint_path` parameter - is an absolute path or not. - - If set to False, the value of `endpoint_path` will be appended to `self.base_url` using - [`urllib.parse.urljoin()`](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin) - function. - - If set to True, base url will be overridden and the value of the `endpoint_path` will - be used instead. - cookies: Dict or CookieJar object of cookies to send with the request - files: Dictionary of 'name': file-like-objects (or {'name': file-tuple}) for multipart encoding upload. - ignore_auth: Boolean marking, whether the default auth_header should be ignored. - **kwargs: All other keyword arguments supported by - [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). - - Returns: - A httpx.Response object. - """ - - method = 'DELETE' - r = await self._request_raw(method, endpoint_path, params=params, headers=headers, data=data, json=json, - cookies=cookies, is_absolute_path=is_absolute_path, files=files, - ignore_auth=ignore_auth, **kwargs) - return r - - def __enter__(self): + base_url (str): The base URL for the API. + retries (int, optional): The maximum number of retries for failed requests. Defaults to 3. + timeout (Optional[float], optional): The request timeout in seconds. Defaults to None. + verify_ssl (bool, optional): Enable or disable SSL verification. Defaults to True. + retry_status_codes (Optional[List[int]], optional): List of status codes to retry on. Defaults to None. + rate_limit (Optional[int], optional): Maximum number of concurrent requests. Defaults to None. + default_params (Optional[Dict[str, str]], optional): Default query parameters for each request. + Defaults to None. + auth_header (Optional[Dict[str, str]], optional): Authentication header for each request. Defaults to None. + user_agent (Optional[str], optional): Custom user-agent string for each request. Defaults to None. + """ + self.base_url = base_url if base_url.endswith("/") else base_url + "/" + self.retries = retries + self.timeout = httpx.Timeout(timeout) if timeout else None + self.verify_ssl = verify_ssl + self.retry_status_codes = retry_status_codes or [] + self.rate_limit = rate_limit + self.default_params = default_params or {} + self.auth_header = auth_header or {} + self.semaphore = asyncio.Semaphore(rate_limit) if rate_limit else None + self.client = httpx.AsyncClient(timeout=self.timeout, verify=self.verify_ssl, + headers={"User-Agent": user_agent} if user_agent else None) + + async def __aenter__(self): + await self.client.__aenter__() return self - def __exit__(self, exc_type, exc_val, exc_tb): - pass + async def __aexit__(self, *args): + await self.client.__aexit__(*args) + + async def _request( + self, + method: str, + endpoint: Optional[str] = None, + params: Optional[Dict[str, str]] = None, + headers: Optional[Dict[str, str]] = None, + json: Optional[Dict[str, Any]] = None, + ) -> httpx.Response: + url = urljoin(self.base_url, endpoint or "") + all_params = {**self.default_params, **(params or {})} + all_headers = {**self.auth_header, **(headers or {})} + + for retry_attempt in range(self.retries + 1): + try: + if self.semaphore: + async with self.semaphore: + response = await self.client.request( + method, + url, + params=all_params, + headers=all_headers, + json=json, + ) + else: + response = await self.client.request( + method, + url, + params=all_params, + headers=all_headers, + json=json, + ) + if response.status_code not in self.retry_status_codes: + response.raise_for_status() + return response + except httpx.HTTPError: + if retry_attempt == self.retries: + raise + await asyncio.sleep(2 ** retry_attempt) + + async def get(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + return await self._request("GET", endpoint, **kwargs) + + async def post(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + return await self._request("POST", endpoint, **kwargs) + + async def put(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + return await self._request("PUT", endpoint, **kwargs) + + async def patch(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + return await self._request("PATCH", endpoint, **kwargs) + + async def delete(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + return await self._request("DELETE", endpoint, **kwargs) diff --git a/src/keboola/http_client/example_async_poekapi.py b/src/keboola/http_client/example_async_poekapi.py new file mode 100644 index 0000000..89c211c --- /dev/null +++ b/src/keboola/http_client/example_async_poekapi.py @@ -0,0 +1,50 @@ +import requests +import time +import asyncio +from async_client import AsyncHttpClient + + +def fetch_pokemon_details_sync(url: str): + response = requests.get(url) + response.raise_for_status() + return response.json() + + +async def fetch_pokemon_details_async(client: AsyncHttpClient, endpoint: str): + response = await client.get(endpoint) + return response.json() + + +def main_sync(): + base_url = "https://pokeapi.co/api/v2/pokemon/" + start_time = time.time() + + pokemon_details = [] + for i in range(1, 152): + url = f"{base_url}{i}" + details = fetch_pokemon_details_sync(url) + pokemon_details.append(details) + + end_time = time.time() + print(f"Sync: Fetched details for {len(pokemon_details)} Pokémon in {end_time - start_time:.2f} seconds.") + + +async def main_async(): + base_url = "https://pokeapi.co/api/v2/pokemon/" + start_time = time.time() + + async def fetch_pokemon(client, i): + endpoint = f"{i}" + details = await fetch_pokemon_details_async(client, endpoint) + return details + + async with AsyncHttpClient(base_url) as client: + pokemon_details = await asyncio.gather(*(fetch_pokemon(client, i) for i in range(1, 152))) + + end_time = time.time() + print(f"Async: Fetched details for {len(pokemon_details)} Pokémon in {end_time - start_time:.2f} seconds.") + + +if __name__ == "__main__": + main_sync() + asyncio.run(main_async()) diff --git a/src/keboola/http_client/example_async_trivia.py b/src/keboola/http_client/example_async_trivia.py new file mode 100644 index 0000000..ea02f4e --- /dev/null +++ b/src/keboola/http_client/example_async_trivia.py @@ -0,0 +1,27 @@ +import asyncio +from async_client import AsyncHttpClient + + +async def fetch_trivia_questions(client: AsyncHttpClient, category_id: int, amount: int = 10) -> dict: + response = await client.get("api.php", params={"amount": amount, "category": category_id, "type": "multiple"}) + data = response.json() + return data["results"] + + +async def main(): + base_url = "https://opentdb.com/" + client = AsyncHttpClient(base_url) + + # Category IDs for General Knowledge, Science & Nature, and Sports + categories = [9, 17, 21] + + tasks = [fetch_trivia_questions(client, category_id) for category_id in categories] + trivia_questions = await asyncio.gather(*tasks) + + for idx, questions in enumerate(trivia_questions): + print(f"Category {categories[idx]}:") + for q in questions: + print(f" - {q['question']}") + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/keboola/http_client/test_async.py b/src/keboola/http_client/test_async.py deleted file mode 100644 index 87c0556..0000000 --- a/src/keboola/http_client/test_async.py +++ /dev/null @@ -1,32 +0,0 @@ -from async_client import AsyncHttpClient -import requests -import asyncio -import time - -common_url = "https://pokeapi.co/api/v2/" -nr_of_pokemons = 202 - - -async def async_get(url, nr_of_pokemons): - pokemons = [] - client = AsyncHttpClient(url) - with client: - for n in range(1, nr_of_pokemons): - r = await client.get(f"pokemon/{str(n)}") - pokemons.append(r.json()['name']) - - -def normal_get(url, nr_of_pokemons): - pokemons = [] - for n in range(1, nr_of_pokemons): - r = requests.get(url + f"pokemon/{str(n)}") - pokemons.append(r.json()['name']) - - -start_time = time.time() -asyncio.run(async_get(common_url, nr_of_pokemons)) -print(f"Async took {time.time() - start_time} seconds.") - -start_time = time.time() -normal_get(common_url, nr_of_pokemons) -print(f"Normal took {time.time() - start_time} seconds.") diff --git a/tests/test_async.py b/tests/test_async.py new file mode 100644 index 0000000..1ff1a84 --- /dev/null +++ b/tests/test_async.py @@ -0,0 +1,82 @@ +import pytest +from unittest.mock import AsyncMock, MagicMock +from async_client import AsyncHttpClient + +import keboola.http_client.async_client.AsyncHttpClient + + +@pytest.mark.asyncio +async def test_post_raw_default_pars_with_none_custom_pars_passes(): + url = 'http://example.com/' + test_def_par = {"default_par": "test"} + cl = AsyncHttpClient(url, default_params=test_def_par) + + cl.client.request = AsyncMock() + await cl.post_raw() + + cl.client.request.assert_called_with('POST', url, params=test_def_par, headers=None, json=None, content=None) + + await cl.aclose() + + +@pytest.mark.asyncio +async def test_post_default_pars_with_none_custom_pars_passes(): + url = 'http://example.com/' + test_def_par = {"default_par": "test"} + cl = AsyncHttpClient(url, default_params=test_def_par) + + cl.client.request = AsyncMock() + await cl.post() + + cl.client.request.assert_called_with('POST', url, params=test_def_par, headers=None, json=None, content=None) + + await cl.aclose() + + +@pytest.mark.asyncio +async def test_post_raw_default_pars_with_custom_pars_passes(): + url = 'http://example.com/' + test_def_par = {"default_par": "test"} + cl = AsyncHttpClient(url, default_params=test_def_par) + + cust_par = {"custom_par": "custom_par_value"} + cl.client.request = AsyncMock() + await cl.post_raw(params=cust_par) + + test_cust_def_par = {**test_def_par, **cust_par} + cl.client.request.assert_called_with('POST', url, params=test_cust_def_par, headers=None, json=None, content=None) + + await cl.aclose() + + +@pytest.mark.asyncio +async def test_post_default_pars_with_custom_pars_passes(): + url = 'http://example.com/' + test_def_par = {"default_par": "test"} + cl = AsyncHttpClient(url, default_params=test_def_par) + + cust_par = {"custom_par": "custom_par_value"} + cl.client.request = AsyncMock() + await cl.post(params=cust_par) + + test_cust_def_par = {**test_def_par, **cust_par} + cl.client.request.assert_called_with('POST', url, params=test_cust_def_par, headers=None, json=None, content=None) + + await cl.aclose() + + +@pytest.mark.asyncio +async def test_post_raw_default_pars_with_custom_pars_to_None_passes(): + url = 'http://example.com/' + test_def_par = {"default_par": "test"} + cl = AsyncHttpClient(url, default_params=test_def_par) + + cust_par = None + cl.client.request = AsyncMock() + await cl.post_raw(params=cust_par) + + _cust_par_transformed = {} + test_cust_def_par = {**test_def_par, **_cust_par_transformed} + cl.client.request.assert_called_with('POST', url, params=test_cust_def_par, headers=None, json=None, content=None) + + await cl.aclose() From 00546a9a808a47e219d679ac16d4ef6aa3859fd9 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Fri, 21 Apr 2023 16:38:47 +0200 Subject: [PATCH 06/56] add raw methods --- .github/workflows/deploy_to_test.yml | 2 +- requirements.txt | 3 +- src/keboola/http_client/async_client.py | 37 ++++++++++++++++--- .../http_client/example_async_poekapi.py | 2 +- .../http_client/example_async_trivia.py | 22 +++++------ tests/test_async.py | 4 +- 6 files changed, 45 insertions(+), 25 deletions(-) diff --git a/.github/workflows/deploy_to_test.yml b/.github/workflows/deploy_to_test.yml index 0a61c7f..4b18403 100644 --- a/.github/workflows/deploy_to_test.yml +++ b/.github/workflows/deploy_to_test.yml @@ -20,7 +20,7 @@ jobs: run: | python -m pip install --upgrade pip pip install setuptools wheel twine - pip install flake8 pytest + pip install flake8 pytest pytest-asyncio # Add pytest-asyncio here pip install -r requirements.txt - name: Lint with flake8 run: | diff --git a/requirements.txt b/requirements.txt index 7ffd6c3..a0c12bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ requests -httpx==0.20.0 -async_lru \ No newline at end of file +httpx==0.24.0 \ No newline at end of file diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 34e612c..68a21a0 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -94,17 +94,42 @@ async def _request( raise await asyncio.sleep(2 ** retry_attempt) - async def get(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + async def get(self, endpoint: Optional[str] = None, **kwargs) -> Any: + response = await self._get_raw(endpoint, **kwargs) + response.raise_for_status() + return response.json() + + async def _get_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: return await self._request("GET", endpoint, **kwargs) - async def post(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + async def post(self, endpoint: Optional[str] = None, **kwargs) -> Any: + response = await self._post_raw(endpoint, **kwargs) + response.raise_for_status() + return response.json() + + async def _post_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: return await self._request("POST", endpoint, **kwargs) - async def put(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + async def put(self, endpoint: Optional[str] = None, **kwargs) -> Any: + response = await self._put_raw(endpoint, **kwargs) + response.raise_for_status() + return response.json() + + async def _put_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: return await self._request("PUT", endpoint, **kwargs) - async def patch(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + async def patch(self, endpoint: Optional[str] = None, **kwargs) -> Any: + response = await self._patch_raw(endpoint, **kwargs) + response.raise_for_status() + return response.json() + + async def _patch_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: return await self._request("PATCH", endpoint, **kwargs) - async def delete(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: - return await self._request("DELETE", endpoint, **kwargs) + async def delete(self, endpoint: Optional[str] = None, **kwargs) -> Any: + response = await self._delete_raw(endpoint, **kwargs) + response.raise_for_status() + return response.json() + + async def _delete_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + return await self._request("DELETE", endpoint, **kwargs) \ No newline at end of file diff --git a/src/keboola/http_client/example_async_poekapi.py b/src/keboola/http_client/example_async_poekapi.py index 89c211c..50a2a7f 100644 --- a/src/keboola/http_client/example_async_poekapi.py +++ b/src/keboola/http_client/example_async_poekapi.py @@ -12,7 +12,7 @@ def fetch_pokemon_details_sync(url: str): async def fetch_pokemon_details_async(client: AsyncHttpClient, endpoint: str): response = await client.get(endpoint) - return response.json() + return response def main_sync(): diff --git a/src/keboola/http_client/example_async_trivia.py b/src/keboola/http_client/example_async_trivia.py index ea02f4e..b5780a4 100644 --- a/src/keboola/http_client/example_async_trivia.py +++ b/src/keboola/http_client/example_async_trivia.py @@ -4,24 +4,22 @@ async def fetch_trivia_questions(client: AsyncHttpClient, category_id: int, amount: int = 10) -> dict: response = await client.get("api.php", params={"amount": amount, "category": category_id, "type": "multiple"}) - data = response.json() - return data["results"] + return response["results"] async def main(): base_url = "https://opentdb.com/" - client = AsyncHttpClient(base_url) + async with AsyncHttpClient(base_url) as client: + # Category IDs for General Knowledge, Science & Nature, and Sports + categories = [9, 17, 21] - # Category IDs for General Knowledge, Science & Nature, and Sports - categories = [9, 17, 21] + tasks = [fetch_trivia_questions(client, category_id) for category_id in categories] + trivia_questions = await asyncio.gather(*tasks) - tasks = [fetch_trivia_questions(client, category_id) for category_id in categories] - trivia_questions = await asyncio.gather(*tasks) - - for idx, questions in enumerate(trivia_questions): - print(f"Category {categories[idx]}:") - for q in questions: - print(f" - {q['question']}") + for idx, questions in enumerate(trivia_questions): + print(f"Category {categories[idx]}:") + for q in questions: + print(f" - {q['question']}") if __name__ == "__main__": asyncio.run(main()) diff --git a/tests/test_async.py b/tests/test_async.py index 1ff1a84..e63c066 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -1,8 +1,6 @@ import pytest from unittest.mock import AsyncMock, MagicMock -from async_client import AsyncHttpClient - -import keboola.http_client.async_client.AsyncHttpClient +from keboola.http_client.async_client import AsyncHttpClient @pytest.mark.asyncio From 7006391616158c244d66f58f4cef35394859ab6f Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Fri, 21 Apr 2023 16:42:35 +0200 Subject: [PATCH 07/56] update client reference --- tests/test_async.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_async.py b/tests/test_async.py index e63c066..add6c7d 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -10,11 +10,11 @@ async def test_post_raw_default_pars_with_none_custom_pars_passes(): cl = AsyncHttpClient(url, default_params=test_def_par) cl.client.request = AsyncMock() - await cl.post_raw() + await cl._post_raw() cl.client.request.assert_called_with('POST', url, params=test_def_par, headers=None, json=None, content=None) - await cl.aclose() + await cl.client.aclose() @pytest.mark.asyncio @@ -28,7 +28,7 @@ async def test_post_default_pars_with_none_custom_pars_passes(): cl.client.request.assert_called_with('POST', url, params=test_def_par, headers=None, json=None, content=None) - await cl.aclose() + await cl.client.aclose() @pytest.mark.asyncio @@ -39,12 +39,12 @@ async def test_post_raw_default_pars_with_custom_pars_passes(): cust_par = {"custom_par": "custom_par_value"} cl.client.request = AsyncMock() - await cl.post_raw(params=cust_par) + await cl.client.post_raw(params=cust_par) test_cust_def_par = {**test_def_par, **cust_par} cl.client.request.assert_called_with('POST', url, params=test_cust_def_par, headers=None, json=None, content=None) - await cl.aclose() + await cl.client.aclose() @pytest.mark.asyncio @@ -60,7 +60,7 @@ async def test_post_default_pars_with_custom_pars_passes(): test_cust_def_par = {**test_def_par, **cust_par} cl.client.request.assert_called_with('POST', url, params=test_cust_def_par, headers=None, json=None, content=None) - await cl.aclose() + await cl.client.aclose() @pytest.mark.asyncio @@ -71,10 +71,10 @@ async def test_post_raw_default_pars_with_custom_pars_to_None_passes(): cust_par = None cl.client.request = AsyncMock() - await cl.post_raw(params=cust_par) + await cl.client.post_raw(params=cust_par) _cust_par_transformed = {} test_cust_def_par = {**test_def_par, **_cust_par_transformed} cl.client.request.assert_called_with('POST', url, params=test_cust_def_par, headers=None, json=None, content=None) - await cl.aclose() + await cl.client.aclose() From c0e2bba573b7535878c2692fa58639fb335af594 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Tue, 23 May 2023 13:39:03 +0200 Subject: [PATCH 08/56] cleanup, readme update --- README.md | 37 ++- src/keboola/http_client/async_client.py | 51 +++-- .../http_client/example_async_poekapi.py | 6 +- src/keboola/http_client/retry_wrapper.py | 214 ------------------ 4 files changed, 75 insertions(+), 233 deletions(-) delete mode 100644 src/keboola/http_client/retry_wrapper.py diff --git a/README.md b/README.md index ea7b864..04ee4ce 100644 --- a/README.md +++ b/README.md @@ -203,4 +203,39 @@ class KBCStorageClient(HttpClient): cl = KBCStorageClient("my_token") print(cl.get_files()) -``` \ No newline at end of file +``` + +## Async Usage + +The package also provides an asynchronous version of the HTTP client called AsyncHttpClient. +It allows you to make asynchronous requests using async/await syntax. To use the AsyncHttpClient, import it from keboola.http_client_async: + +```python +from keboola.http_client_async import AsyncHttpClient +``` + +The AsyncHttpClient class provides similar functionality as the HttpClient class, but with asynchronous methods such as get, post, put, patch, and delete that return awaitable coroutines. +You can use these methods within async functions to perform non-blocking HTTP requests. + +For example, to make an asynchronous GET request: + +```python +import asyncio +from keboola.http_client_async import AsyncHttpClient + +async def main(): + base_url = "https://api.example.com/" + async with AsyncHttpClient(base_url) as client: + response = await client.get("endpoint") + + if response.status_code == 200: + data = response.json() + # Process the response data + else: + # Handle the error + +asyncio.run(main()) +``` + +The AsyncHttpClient provides similar initialization and request methods as the HttpClient, but the request methods return awaitable coroutines that can be awaited in an asynchronous context. +You can use the async with statement to manage the lifecycle of the client and ensure that resources are properly cleaned up. diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 68a21a0..426afb8 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -1,3 +1,4 @@ +import random import httpx import asyncio from typing import Optional, Dict, Any, List @@ -19,10 +20,11 @@ def __init__( rate_limit: Optional[int] = None, default_params: Optional[Dict[str, str]] = None, auth_header: Optional[Dict[str, str]] = None, - user_agent: Optional[str] = None, + default_headers: Optional[Dict[str, str]] = None, + backoff_factor: float = 2.0 ): """ - Initialize the AsyncHttpClientV2 instance. + Initialize the AsyncHttpClient instance. Args: base_url (str): The base URL for the API. @@ -32,9 +34,8 @@ def __init__( retry_status_codes (Optional[List[int]], optional): List of status codes to retry on. Defaults to None. rate_limit (Optional[int], optional): Maximum number of concurrent requests. Defaults to None. default_params (Optional[Dict[str, str]], optional): Default query parameters for each request. - Defaults to None. auth_header (Optional[Dict[str, str]], optional): Authentication header for each request. Defaults to None. - user_agent (Optional[str], optional): Custom user-agent string for each request. Defaults to None. + backoff_factor (float, optional): The backoff factor for retries. Defaults to 2.0. """ self.base_url = base_url if base_url.endswith("/") else base_url + "/" self.retries = retries @@ -45,8 +46,23 @@ def __init__( self.default_params = default_params or {} self.auth_header = auth_header or {} self.semaphore = asyncio.Semaphore(rate_limit) if rate_limit else None - self.client = httpx.AsyncClient(timeout=self.timeout, verify=self.verify_ssl, - headers={"User-Agent": user_agent} if user_agent else None) + self.default_headers = default_headers or {} + self.backoff_factor = backoff_factor + + self.client = httpx.AsyncClient(timeout=self.timeout, verify=self.verify_ssl, headers=self.default_headers) + + async def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False): + # build URL Specification + url_path = str(endpoint_path).strip() if endpoint_path is not None else '' + + if not url_path: + url = self.base_url + elif not is_absolute_path: + url = urljoin(self.base_url, endpoint_path) + else: + url = endpoint_path + + return url async def __aenter__(self): await self.client.__aenter__() @@ -55,17 +71,24 @@ async def __aenter__(self): async def __aexit__(self, *args): await self.client.__aexit__(*args) + async def close(self): + await self.client.aclose() + async def _request( self, method: str, endpoint: Optional[str] = None, - params: Optional[Dict[str, str]] = None, + params: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, json: Optional[Dict[str, Any]] = None, + **kwargs ) -> httpx.Response: - url = urljoin(self.base_url, endpoint or "") + # build URL Specification + is_absolute_path = kwargs.pop('is_absolute_path', False) + url = await self._build_url(endpoint, is_absolute_path) + all_params = {**self.default_params, **(params or {})} - all_headers = {**self.auth_header, **(headers or {})} + all_headers = {**self.auth_header, **self.default_headers, **(headers or {})} # include default headers for retry_attempt in range(self.retries + 1): try: @@ -92,11 +115,11 @@ async def _request( except httpx.HTTPError: if retry_attempt == self.retries: raise - await asyncio.sleep(2 ** retry_attempt) + backoff_factor = 2 ** retry_attempt + await asyncio.sleep(random.uniform(0, backoff_factor)) async def get(self, endpoint: Optional[str] = None, **kwargs) -> Any: response = await self._get_raw(endpoint, **kwargs) - response.raise_for_status() return response.json() async def _get_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: @@ -104,7 +127,6 @@ async def _get_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Resp async def post(self, endpoint: Optional[str] = None, **kwargs) -> Any: response = await self._post_raw(endpoint, **kwargs) - response.raise_for_status() return response.json() async def _post_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: @@ -112,7 +134,6 @@ async def _post_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Res async def put(self, endpoint: Optional[str] = None, **kwargs) -> Any: response = await self._put_raw(endpoint, **kwargs) - response.raise_for_status() return response.json() async def _put_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: @@ -120,7 +141,6 @@ async def _put_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Resp async def patch(self, endpoint: Optional[str] = None, **kwargs) -> Any: response = await self._patch_raw(endpoint, **kwargs) - response.raise_for_status() return response.json() async def _patch_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: @@ -128,8 +148,7 @@ async def _patch_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Re async def delete(self, endpoint: Optional[str] = None, **kwargs) -> Any: response = await self._delete_raw(endpoint, **kwargs) - response.raise_for_status() return response.json() async def _delete_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: - return await self._request("DELETE", endpoint, **kwargs) \ No newline at end of file + return await self._request("DELETE", endpoint, **kwargs) diff --git a/src/keboola/http_client/example_async_poekapi.py b/src/keboola/http_client/example_async_poekapi.py index 50a2a7f..396d42d 100644 --- a/src/keboola/http_client/example_async_poekapi.py +++ b/src/keboola/http_client/example_async_poekapi.py @@ -32,14 +32,16 @@ def main_sync(): async def main_async(): base_url = "https://pokeapi.co/api/v2/pokemon/" start_time = time.time() + client = AsyncHttpClient(base_url=base_url) + async def fetch_pokemon(client, i): endpoint = f"{i}" details = await fetch_pokemon_details_async(client, endpoint) return details - async with AsyncHttpClient(base_url) as client: - pokemon_details = await asyncio.gather(*(fetch_pokemon(client, i) for i in range(1, 152))) + async with client as c: + pokemon_details = await asyncio.gather(*(fetch_pokemon(c, poke_id) for poke_id in range(1, 152))) end_time = time.time() print(f"Async: Fetched details for {len(pokemon_details)} Pokémon in {end_time - start_time:.2f} seconds.") diff --git a/src/keboola/http_client/retry_wrapper.py b/src/keboola/http_client/retry_wrapper.py deleted file mode 100644 index 1bfe74c..0000000 --- a/src/keboola/http_client/retry_wrapper.py +++ /dev/null @@ -1,214 +0,0 @@ -import asyncio -import random -import time -from datetime import datetime -from functools import partial -from http import HTTPStatus -from typing import Any, Callable, Coroutine, Iterable, Mapping, Union, Type - -import httpx -from dateutil.parser import isoparse - - -class RetryTransport(httpx.AsyncBaseTransport): - """ - Credits to: https://github.com/DeepakArora76 - A custom HTTP transport that automatically retries requests using an exponential backoff strategy - for specific HTTP status codes and request methods. - - Args: - max_attempts (int, optional): The maximum number of times to retry a request before giving up. Defaults to 10. - max_backoff_wait (float, optional): The maximum time to wait between retries in seconds. Defaults to 60. - backoff_factor (float, optional): The factor by which the wait time increases with each retry attempt. - Defaults to 0.1. - jitter_ratio (float, optional): The amount of jitter to add to the backoff time. Jitter is a random - value added to the backoff time to avoid a "thundering herd" effect. The value should be between 0 and 0.5. - Defaults to 0.1. - respect_retry_after_header (bool, optional): Whether to respect the Retry-After header in HTTP responses - when deciding how long to wait before retrying. Defaults to True. - retryable_methods (Iterable[str], optional): The HTTP methods that can be retried. Defaults to - ["GET", "PUT", "DELETE", "PATCH"]. - retry_status_codes (Iterable[int], optional): The HTTP status codes that can be retried. Defaults to - [429, 502, 503, 504]. - - Attributes: - _wrapped_transport (httpx.AsyncBaseTransport): The underlying HTTP transport - being wrapped. - _max_attempts (int): The maximum number of times to retry a request. - _backoff_factor (float): The factor by which the wait time increases with each retry attempt. - _respect_retry_after_header (bool): Whether to respect the Retry-After header in HTTP responses. - _retryable_methods (frozenset): The HTTP methods that can be retried. - _retry_status_codes (frozenset): The HTTP status codes that can be retried. - _jitter_ratio (float): The amount of jitter to add to the backoff time. - _max_backoff_wait (float): The maximum time to wait between retries in seconds. - - """ - - RETRYABLE_METHODS = frozenset(["GET", "PUT", "DELETE", "PATCH"]) - RETRYABLE_STATUS_CODES = frozenset( - [ - HTTPStatus.TOO_MANY_REQUESTS, - HTTPStatus.BAD_GATEWAY, - HTTPStatus.SERVICE_UNAVAILABLE, - HTTPStatus.GATEWAY_TIMEOUT, - ] - ) - MAX_BACKOFF_WAIT = 60 - - def __init__( - self, - max_attempts: int = 10, - max_backoff_wait: float = MAX_BACKOFF_WAIT, - backoff_factor: float = 0.1, - jitter_ratio: float = 0.1, - respect_retry_after_header: bool = True, - retryable_methods: Iterable[str] = None, - retry_status_codes: Iterable[int] = None, - ) -> None: - """ - Initializes the instance of RetryTransport class with the given parameters. - - Args: - max_attempts (int, optional): - The maximum number of times the request can be retried in case of failure. - Defaults to 10. - max_backoff_wait (float, optional): - The maximum amount of time (in seconds) to wait before retrying a request. - Defaults to 60. - backoff_factor (float, optional): - The factor by which the waiting time will be multiplied in each retry attempt. - Defaults to 0.1. - jitter_ratio (float, optional): - The ratio of randomness added to the waiting time to prevent simultaneous retries. - Should be between 0 and 0.5. Defaults to 0.1. - respect_retry_after_header (bool, optional): - A flag to indicate if the Retry-After header should be respected. - If True, the waiting time specified in Retry-After header is used for the waiting time. - Defaults to True. - retryable_methods (Iterable[str], optional): - The HTTP methods that can be retried. Defaults to ["GET", "PUT", "DELETE", "PATCH"]. - retry_status_codes (Iterable[int], optional): - The HTTP status codes that can be retried. - Defaults to [429, 502, 503, 504]. - """ - self._wrapped_transport = httpx.AsyncBaseTransport - if jitter_ratio < 0 or jitter_ratio > 0.5: - raise ValueError( - f"Jitter ratio should be between 0 and 0.5, actual {jitter_ratio}" - ) - - self._max_attempts = max_attempts - self._backoff_factor = backoff_factor - self._respect_retry_after_header = respect_retry_after_header - self._retryable_methods = ( - frozenset(retryable_methods) - if retryable_methods - else self.RETRYABLE_METHODS - ) - self._retry_status_codes = ( - frozenset(retry_status_codes) - if retry_status_codes - else self.RETRYABLE_STATUS_CODES - ) - self._jitter_ratio = jitter_ratio - self._max_backoff_wait = max_backoff_wait - - async def handle_async_request(self, request: httpx.Request) -> httpx.Response: - """Sends an HTTP request, possibly with retries. - - Args: - request: The request to perform. - - Returns: - The response. - - """ - transport: Type[httpx.AsyncBaseTransport] = self._wrapped_transport - if request.method in self._retryable_methods: - send_method = partial(transport.handle_async_request) - response = await self._retry_operation_async(request, send_method) - else: - response = await transport.handle_async_request(request) - return response - - async def aclose(self) -> None: - """ - Closes the underlying HTTP transport, terminating all outstanding connections and rejecting any further - requests. - - This should be called before the object is dereferenced, to ensure that connections are properly cleaned up. - """ - transport: Type[httpx.AsyncBaseTransport] = self._wrapped_transport - await transport.aclose(self) - - def _calculate_sleep( - self, attempts_made: int, headers: Union[httpx.Headers, Mapping[str, str]] - ) -> float: - # Retry-After - # The Retry-After response HTTP header indicates how long the user agent should wait before - # making a follow-up request. There are three main cases this header is used: - # - When sent with a 503 (Service Unavailable) response, this indicates how long the service - # is expected to be unavailable. - # - When sent with a 429 (Too Many Requests) response, this indicates how long to wait before - # making a new request. - # - When sent with a redirect response, such as 301 (Moved Permanently), this indicates the - # minimum time that the user agent is asked to wait before issuing the redirected request. - retry_after_header = (headers.get("Retry-After") or "").strip() - if self._respect_retry_after_header and retry_after_header: - if retry_after_header.isdigit(): - return float(retry_after_header) - - try: - parsed_date = isoparse( - retry_after_header - ).astimezone() # converts to local time - diff = (parsed_date - datetime.now().astimezone()).total_seconds() - if diff > 0: - return min(diff, self._max_backoff_wait) - except ValueError: - pass - - backoff = self._backoff_factor * (2 ** (attempts_made - 1)) - jitter = (backoff * self._jitter_ratio) * random.choice([1, -1]) - total_backoff = backoff + jitter - return min(total_backoff, self._max_backoff_wait) - - async def _retry_operation_async( - self, - request: httpx.Request, - send_method: Callable[..., Coroutine[Any, Any, httpx.Response]], - ) -> httpx.Response: - remaining_attempts = self._max_attempts - attempts_made = 0 - while True: - if attempts_made > 0: - await asyncio.sleep(self._calculate_sleep(attempts_made, {})) - response = await send_method(request) - if ( - remaining_attempts < 1 - or response.status_code not in self._retry_status_codes - ): - return response - await response.aclose() - attempts_made += 1 - remaining_attempts -= 1 - - def _retry_operation( - self, - request: httpx.Request, - send_method: Callable[..., httpx.Response], - ) -> httpx.Response: - remaining_attempts = self._max_attempts - attempts_made = 0 - while True: - if attempts_made > 0: - time.sleep(self._calculate_sleep(attempts_made, {})) - response = send_method(request) - if ( - remaining_attempts < 1 - or response.status_code not in self._retry_status_codes - ): - return response - response.close() - attempts_made += 1 - remaining_attempts -= 1 From e98f81be1b97601a4b99734c19efd6f3cdadb448 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Tue, 23 May 2023 16:33:48 +0200 Subject: [PATCH 09/56] move examples to separate folder --- README.md | 12 ++-- docs/examples/async_poekapi_large.py | 68 +++++++++++++++++++ docs/examples/async_pokeapi.py | 55 +++++++++++++++ docs/examples/sync_poekapi.py | 46 +++++++++++++ setup.py | 3 +- src/keboola/http_client/__init__.py | 2 +- .../http_client/example_async_poekapi.py | 52 -------------- .../http_client/example_async_trivia.py | 25 ------- 8 files changed, 177 insertions(+), 86 deletions(-) create mode 100644 docs/examples/async_poekapi_large.py create mode 100644 docs/examples/async_pokeapi.py create mode 100644 docs/examples/sync_poekapi.py delete mode 100644 src/keboola/http_client/example_async_poekapi.py delete mode 100644 src/keboola/http_client/example_async_trivia.py diff --git a/README.md b/README.md index 04ee4ce..2963949 100644 --- a/README.md +++ b/README.md @@ -211,17 +211,15 @@ The package also provides an asynchronous version of the HTTP client called Asyn It allows you to make asynchronous requests using async/await syntax. To use the AsyncHttpClient, import it from keboola.http_client_async: ```python -from keboola.http_client_async import AsyncHttpClient +from keboola.http_client import AsyncHttpClient ``` -The AsyncHttpClient class provides similar functionality as the HttpClient class, but with asynchronous methods such as get, post, put, patch, and delete that return awaitable coroutines. +The AsyncHttpClient class provides similar functionality as the HttpClient class, but with asynchronous methods such as get, post, put, patch, and delete that return awaitable coroutines. You can use these methods within async functions to perform non-blocking HTTP requests. -For example, to make an asynchronous GET request: - ```python import asyncio -from keboola.http_client_async import AsyncHttpClient +from keboola.http_client import AsyncHttpClient async def main(): base_url = "https://api.example.com/" @@ -237,5 +235,5 @@ async def main(): asyncio.run(main()) ``` -The AsyncHttpClient provides similar initialization and request methods as the HttpClient, but the request methods return awaitable coroutines that can be awaited in an asynchronous context. -You can use the async with statement to manage the lifecycle of the client and ensure that resources are properly cleaned up. +The AsyncHttpClient provides similar initialization and request methods as the HttpClient. +The request methods return awaitable coroutines that can be awaited in an asynchronous context. diff --git a/docs/examples/async_poekapi_large.py b/docs/examples/async_poekapi_large.py new file mode 100644 index 0000000..bbd0859 --- /dev/null +++ b/docs/examples/async_poekapi_large.py @@ -0,0 +1,68 @@ +import time +import asyncio +from keboola.http_client import AsyncHttpClient +import csv +import httpx + +async def fetch_pokemon_details_async(client: AsyncHttpClient, endpoint: str): + r = await client.get(endpoint) + return r + +async def main_async(): + base_url = "https://pokeapi.co/api/v2/pokemon/" + start_time = time.time() + client = AsyncHttpClient(base_url=base_url) + + # Fetch Pokemon Details + + async def fetch_pokemon(client, poke_id): + try: + r = await fetch_pokemon_details_async(client, str(poke_id)) + return r + except httpx.HTTPStatusError as e: + if e.response.status_code == 404: + return None + else: + raise + + async with client as c: + poke_id = 1 + + # Check if the file exists and write the header if it doesn't + filename = "pokemon_details.csv" + fieldnames = ["name", "height", "weight"] + + try: + with open(filename, "x", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + except FileExistsError: + pass + + while True: + details = await fetch_pokemon(c, poke_id) + if details is None: + break + + await save_to_csv(details) + + poke_id += 1 + + end_time = time.time() + print(f"Async: Fetched details for {poke_id - 1} Pokémon in {end_time - start_time:.2f} seconds.") + +async def save_to_csv(details): + filename = "pokemon_details.csv" + fieldnames = ["name", "height", "weight"] + + with open(filename, "a", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + + writer.writerow({ + "name": details["name"], + "height": details["height"], + "weight": details["weight"] + }) + +if __name__ == "__main__": + asyncio.run(main_async()) diff --git a/docs/examples/async_pokeapi.py b/docs/examples/async_pokeapi.py new file mode 100644 index 0000000..1cb74b4 --- /dev/null +++ b/docs/examples/async_pokeapi.py @@ -0,0 +1,55 @@ +import time +import asyncio +from keboola.http_client import AsyncHttpClient +import csv +import httpx + +async def fetch_pokemon_details_async(client: AsyncHttpClient, endpoint: str): + r = await client.get(endpoint) + return r + +async def main_async(): + base_url = "https://pokeapi.co/api/v2/pokemon/" + start_time = time.time() + client = AsyncHttpClient(base_url=base_url) + + async def fetch_pokemon(client, poke_id): + try: + r = await fetch_pokemon_details_async(client, str(poke_id)) + return r + except httpx.HTTPStatusError as e: + if e.response.status_code == 404: + return None + else: + raise + + async with client as c: + pokemon_details = [] + poke_id = 1 + + while True: + details = await fetch_pokemon(c, poke_id) + if details is None: + break + pokemon_details.append(details) + poke_id += 1 + + # Save details to CSV + filename = "pokemon_details.csv" + fieldnames = ["name", "height", "weight"] # Define the fields you want to store + + with open(filename, "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + for details in pokemon_details: + writer.writerow({ + "name": details["name"], + "height": details["height"], + "weight": details["weight"] + }) + + end_time = time.time() + print(f"Async: Fetched details for {len(pokemon_details)} Pokémon in {end_time - start_time:.2f} seconds.") + +if __name__ == "__main__": + asyncio.run(main_async()) diff --git a/docs/examples/sync_poekapi.py b/docs/examples/sync_poekapi.py new file mode 100644 index 0000000..1f58487 --- /dev/null +++ b/docs/examples/sync_poekapi.py @@ -0,0 +1,46 @@ +import requests +import time +import csv + +def fetch_pokemon_details_sync(url: str): + r = requests.get(url) + r.raise_for_status() + return r.json() + +def main_sync(): + base_url = "https://pokeapi.co/api/v2/pokemon/" + start_time = time.time() + pokemon_details = [] + poke_id = 1 + + while True: + url = f"{base_url}{poke_id}" + try: + details = fetch_pokemon_details_sync(url) + pokemon_details.append(details) + poke_id += 1 + except requests.exceptions.HTTPError as e: + if e.response.status_code == 404: + break + else: + raise + + # Save details to CSV + filename = "pokemon_details.csv" + fieldnames = ["name", "height", "weight"] # Define the fields you want to store + + with open(filename, "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + for details in pokemon_details: + writer.writerow({ + "name": details["name"], + "height": details["height"], + "weight": details["weight"] + }) + + end_time = time.time() + print(f"Fetched details for {len(pokemon_details)} Pokémon in {end_time - start_time:.2f} seconds.") + +if __name__ == "__main__": + main_sync() diff --git a/setup.py b/setup.py index 72b8c44..7999ea4 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,8 @@ setup_requires=['pytest-runner', 'flake8'], tests_require=['pytest'], install_requires=[ - 'requests' + 'requests', + 'httpx' ], author_email="support@keboola.com", description="General HTTP requests library for Python applications running in Keboola Connection environment", diff --git a/src/keboola/http_client/__init__.py b/src/keboola/http_client/__init__.py index b6ab6da..05f88ab 100644 --- a/src/keboola/http_client/__init__.py +++ b/src/keboola/http_client/__init__.py @@ -1,2 +1,2 @@ from .client import HttpClient # noqa -from .async_client import AsyncHttpClient +from .async_client import AsyncHttpClient # noqa diff --git a/src/keboola/http_client/example_async_poekapi.py b/src/keboola/http_client/example_async_poekapi.py deleted file mode 100644 index 396d42d..0000000 --- a/src/keboola/http_client/example_async_poekapi.py +++ /dev/null @@ -1,52 +0,0 @@ -import requests -import time -import asyncio -from async_client import AsyncHttpClient - - -def fetch_pokemon_details_sync(url: str): - response = requests.get(url) - response.raise_for_status() - return response.json() - - -async def fetch_pokemon_details_async(client: AsyncHttpClient, endpoint: str): - response = await client.get(endpoint) - return response - - -def main_sync(): - base_url = "https://pokeapi.co/api/v2/pokemon/" - start_time = time.time() - - pokemon_details = [] - for i in range(1, 152): - url = f"{base_url}{i}" - details = fetch_pokemon_details_sync(url) - pokemon_details.append(details) - - end_time = time.time() - print(f"Sync: Fetched details for {len(pokemon_details)} Pokémon in {end_time - start_time:.2f} seconds.") - - -async def main_async(): - base_url = "https://pokeapi.co/api/v2/pokemon/" - start_time = time.time() - client = AsyncHttpClient(base_url=base_url) - - - async def fetch_pokemon(client, i): - endpoint = f"{i}" - details = await fetch_pokemon_details_async(client, endpoint) - return details - - async with client as c: - pokemon_details = await asyncio.gather(*(fetch_pokemon(c, poke_id) for poke_id in range(1, 152))) - - end_time = time.time() - print(f"Async: Fetched details for {len(pokemon_details)} Pokémon in {end_time - start_time:.2f} seconds.") - - -if __name__ == "__main__": - main_sync() - asyncio.run(main_async()) diff --git a/src/keboola/http_client/example_async_trivia.py b/src/keboola/http_client/example_async_trivia.py deleted file mode 100644 index b5780a4..0000000 --- a/src/keboola/http_client/example_async_trivia.py +++ /dev/null @@ -1,25 +0,0 @@ -import asyncio -from async_client import AsyncHttpClient - - -async def fetch_trivia_questions(client: AsyncHttpClient, category_id: int, amount: int = 10) -> dict: - response = await client.get("api.php", params={"amount": amount, "category": category_id, "type": "multiple"}) - return response["results"] - - -async def main(): - base_url = "https://opentdb.com/" - async with AsyncHttpClient(base_url) as client: - # Category IDs for General Knowledge, Science & Nature, and Sports - categories = [9, 17, 21] - - tasks = [fetch_trivia_questions(client, category_id) for category_id in categories] - trivia_questions = await asyncio.gather(*tasks) - - for idx, questions in enumerate(trivia_questions): - print(f"Category {categories[idx]}:") - for q in questions: - print(f" - {q['question']}") - -if __name__ == "__main__": - asyncio.run(main()) From 2065246dac0c893222d3a1d2fcf41c7a749d2624 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Wed, 24 May 2023 12:05:09 +0200 Subject: [PATCH 10/56] add new tests --- src/keboola/http_client/async_client.py | 20 +-- tests/test_async.py | 161 ++++++++++++------------ 2 files changed, 91 insertions(+), 90 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 426afb8..d301a0a 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -119,36 +119,36 @@ async def _request( await asyncio.sleep(random.uniform(0, backoff_factor)) async def get(self, endpoint: Optional[str] = None, **kwargs) -> Any: - response = await self._get_raw(endpoint, **kwargs) + response = await self.get_raw(endpoint, **kwargs) return response.json() - async def _get_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + async def get_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: return await self._request("GET", endpoint, **kwargs) async def post(self, endpoint: Optional[str] = None, **kwargs) -> Any: - response = await self._post_raw(endpoint, **kwargs) + response = await self.post_raw(endpoint, **kwargs) return response.json() - async def _post_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + async def post_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: return await self._request("POST", endpoint, **kwargs) async def put(self, endpoint: Optional[str] = None, **kwargs) -> Any: - response = await self._put_raw(endpoint, **kwargs) + response = await self.put_raw(endpoint, **kwargs) return response.json() - async def _put_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + async def put_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: return await self._request("PUT", endpoint, **kwargs) async def patch(self, endpoint: Optional[str] = None, **kwargs) -> Any: - response = await self._patch_raw(endpoint, **kwargs) + response = await self.patch_raw(endpoint, **kwargs) return response.json() - async def _patch_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + async def patch_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: return await self._request("PATCH", endpoint, **kwargs) async def delete(self, endpoint: Optional[str] = None, **kwargs) -> Any: - response = await self._delete_raw(endpoint, **kwargs) + response = await self.delete_raw(endpoint, **kwargs) return response.json() - async def _delete_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + async def delete_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: return await self._request("DELETE", endpoint, **kwargs) diff --git a/tests/test_async.py b/tests/test_async.py index add6c7d..63576ca 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -1,80 +1,81 @@ -import pytest -from unittest.mock import AsyncMock, MagicMock -from keboola.http_client.async_client import AsyncHttpClient - - -@pytest.mark.asyncio -async def test_post_raw_default_pars_with_none_custom_pars_passes(): - url = 'http://example.com/' - test_def_par = {"default_par": "test"} - cl = AsyncHttpClient(url, default_params=test_def_par) - - cl.client.request = AsyncMock() - await cl._post_raw() - - cl.client.request.assert_called_with('POST', url, params=test_def_par, headers=None, json=None, content=None) - - await cl.client.aclose() - - -@pytest.mark.asyncio -async def test_post_default_pars_with_none_custom_pars_passes(): - url = 'http://example.com/' - test_def_par = {"default_par": "test"} - cl = AsyncHttpClient(url, default_params=test_def_par) - - cl.client.request = AsyncMock() - await cl.post() - - cl.client.request.assert_called_with('POST', url, params=test_def_par, headers=None, json=None, content=None) - - await cl.client.aclose() - - -@pytest.mark.asyncio -async def test_post_raw_default_pars_with_custom_pars_passes(): - url = 'http://example.com/' - test_def_par = {"default_par": "test"} - cl = AsyncHttpClient(url, default_params=test_def_par) - - cust_par = {"custom_par": "custom_par_value"} - cl.client.request = AsyncMock() - await cl.client.post_raw(params=cust_par) - - test_cust_def_par = {**test_def_par, **cust_par} - cl.client.request.assert_called_with('POST', url, params=test_cust_def_par, headers=None, json=None, content=None) - - await cl.client.aclose() - - -@pytest.mark.asyncio -async def test_post_default_pars_with_custom_pars_passes(): - url = 'http://example.com/' - test_def_par = {"default_par": "test"} - cl = AsyncHttpClient(url, default_params=test_def_par) - - cust_par = {"custom_par": "custom_par_value"} - cl.client.request = AsyncMock() - await cl.post(params=cust_par) - - test_cust_def_par = {**test_def_par, **cust_par} - cl.client.request.assert_called_with('POST', url, params=test_cust_def_par, headers=None, json=None, content=None) - - await cl.client.aclose() - - -@pytest.mark.asyncio -async def test_post_raw_default_pars_with_custom_pars_to_None_passes(): - url = 'http://example.com/' - test_def_par = {"default_par": "test"} - cl = AsyncHttpClient(url, default_params=test_def_par) - - cust_par = None - cl.client.request = AsyncMock() - await cl.client.post_raw(params=cust_par) - - _cust_par_transformed = {} - test_cust_def_par = {**test_def_par, **_cust_par_transformed} - cl.client.request.assert_called_with('POST', url, params=test_cust_def_par, headers=None, json=None, content=None) - - await cl.client.aclose() +import unittest +from unittest.mock import patch +import httpx +from keboola.http_client import AsyncHttpClient + +class TestAsyncHttpClient(unittest.IsolatedAsyncioTestCase): + base_url = "https://api.example.com" + + def setUp(self): + self.client = AsyncHttpClient(self.base_url) + + async def tearDown(self): + await self.client.close() + + async def test_get(self): + expected_response = {"message": "Success"} + mock_response = httpx.Response(200, json=expected_response) + mock_response._request = httpx.Request("GET", "https://api.example.com/endpoint") + + with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: + response = await self.client.get("/endpoint") + self.assertEqual(response, expected_response) + mock_request.assert_called_once_with("GET", "https://api.example.com/endpoint", params={}, headers={}, json=None) + + async def test_post(self): + expected_response = {"message": "Success"} + mock_response = httpx.Response(200, json=expected_response) + mock_response._request = httpx.Request("POST", "https://api.example.com/endpoint") + + with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: + response = await self.client.post("/endpoint", json={"data": "example"}) + self.assertEqual(response, expected_response) + mock_request.assert_called_once_with("POST", "https://api.example.com/endpoint", params={}, headers={}, json={"data": "example"}) + + async def test_handle_success_response(self): + expected_response = {"message": "Success"} + mock_response = httpx.Response(200, json=expected_response) + mock_response._request = httpx.Request("GET", "https://api.example.com/endpoint") + + with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: + response = await self.client.get("/endpoint") + self.assertEqual(response, expected_response) + mock_request.assert_called_once_with("GET", "https://api.example.com/endpoint", params={}, headers={}, json=None) + + async def test_handle_client_error_response(self): + mock_response = httpx.Response(404) + mock_response._request = httpx.Request("GET", "https://api.example.com/endpoint") + + with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: + with self.assertRaises(httpx.HTTPStatusError): + await self.client.get("/endpoint") + mock_request.assert_called_with("GET", "https://api.example.com/endpoint", params={}, headers={}, json=None) + + async def test_handle_server_error_response(self): + mock_response = httpx.Response(500) + mock_response._request = httpx.Request("GET", "https://api.example.com/endpoint") + + with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: + with self.assertRaises(httpx.HTTPStatusError): + await self.client.get("/endpoint") + mock_request.assert_called_with("GET", "https://api.example.com/endpoint", params={}, headers={}, json=None) + + @patch.object(httpx.AsyncClient, 'request') + async def test_post_raw_default_pars_with_none_custom_pars_passes(self, mock_request): + url = f"{self.base_url}/endpoint" + test_def_par = {"default_par": "test"} + + await self.client.post_raw("/endpoint", params=test_def_par) + + mock_request.assert_called_once_with( + "POST", + url, + params=test_def_par, + headers={}, + json=None + ) + + + +if __name__ == "__main__": + unittest.main() From 331037959ad37626f3e324f0c5c4b91d033aae25 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Wed, 24 May 2023 16:57:31 +0200 Subject: [PATCH 11/56] update tests --- docs/examples/async_poekapi_large.py | 2 - src/keboola/http_client/async_client.py | 33 +++--- tests/test_async.py | 134 +++++++++++++++++++----- 3 files changed, 127 insertions(+), 42 deletions(-) diff --git a/docs/examples/async_poekapi_large.py b/docs/examples/async_poekapi_large.py index bbd0859..188340c 100644 --- a/docs/examples/async_poekapi_large.py +++ b/docs/examples/async_poekapi_large.py @@ -13,8 +13,6 @@ async def main_async(): start_time = time.time() client = AsyncHttpClient(base_url=base_url) - # Fetch Pokemon Details - async def fetch_pokemon(client, poke_id): try: r = await fetch_pokemon_details_async(client, str(poke_id)) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index d301a0a..59291d4 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -83,35 +83,36 @@ async def _request( json: Optional[Dict[str, Any]] = None, **kwargs ) -> httpx.Response: - # build URL Specification + is_absolute_path = kwargs.pop('is_absolute_path', False) url = await self._build_url(endpoint, is_absolute_path) all_params = {**self.default_params, **(params or {})} - all_headers = {**self.auth_header, **self.default_headers, **(headers or {})} # include default headers + all_headers = {**self.auth_header, **self.default_headers, **(headers or {})} + + request_kwargs = { + "method": method, + "url": url, + "params": all_params, + "headers": all_headers + } + + if json is not None: + request_kwargs["json"] = json for retry_attempt in range(self.retries + 1): try: if self.semaphore: async with self.semaphore: - response = await self.client.request( - method, - url, - params=all_params, - headers=all_headers, - json=json, - ) + response = await self.client.request(**request_kwargs) else: - response = await self.client.request( - method, - url, - params=all_params, - headers=all_headers, - json=json, - ) + response = await self.client.request(**request_kwargs) + if response.status_code not in self.retry_status_codes: response.raise_for_status() + return response + except httpx.HTTPError: if retry_attempt == self.retries: raise diff --git a/tests/test_async.py b/tests/test_async.py index 63576ca..d983276 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -5,76 +5,162 @@ class TestAsyncHttpClient(unittest.IsolatedAsyncioTestCase): base_url = "https://api.example.com" - - def setUp(self): - self.client = AsyncHttpClient(self.base_url) - - async def tearDown(self): - await self.client.close() + retries = 3 async def test_get(self): expected_response = {"message": "Success"} mock_response = httpx.Response(200, json=expected_response) mock_response._request = httpx.Request("GET", "https://api.example.com/endpoint") + client = AsyncHttpClient(self.base_url, retries=self.retries) + with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: - response = await self.client.get("/endpoint") + response = await client.get("/endpoint") self.assertEqual(response, expected_response) - mock_request.assert_called_once_with("GET", "https://api.example.com/endpoint", params={}, headers={}, json=None) + mock_request.assert_called_once_with(method="GET", url="https://api.example.com/endpoint", params={}, + headers={}) async def test_post(self): expected_response = {"message": "Success"} mock_response = httpx.Response(200, json=expected_response) mock_response._request = httpx.Request("POST", "https://api.example.com/endpoint") + client = AsyncHttpClient(self.base_url, retries=self.retries) + with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: - response = await self.client.post("/endpoint", json={"data": "example"}) + response = await client.post("/endpoint", json={"data": "example"}) self.assertEqual(response, expected_response) - mock_request.assert_called_once_with("POST", "https://api.example.com/endpoint", params={}, headers={}, json={"data": "example"}) + mock_request.assert_called_once_with(method="POST", url="https://api.example.com/endpoint", params={}, + headers={}, json={"data": "example"}) async def test_handle_success_response(self): expected_response = {"message": "Success"} mock_response = httpx.Response(200, json=expected_response) mock_response._request = httpx.Request("GET", "https://api.example.com/endpoint") + client = AsyncHttpClient(self.base_url, retries=self.retries) + with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: - response = await self.client.get("/endpoint") + response = await client.get("/endpoint") self.assertEqual(response, expected_response) - mock_request.assert_called_once_with("GET", "https://api.example.com/endpoint", params={}, headers={}, json=None) + mock_request.assert_called_once_with(method="GET", url="https://api.example.com/endpoint", params={}, + headers={}) async def test_handle_client_error_response(self): mock_response = httpx.Response(404) mock_response._request = httpx.Request("GET", "https://api.example.com/endpoint") + client = AsyncHttpClient(self.base_url, retries=self.retries) + with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: with self.assertRaises(httpx.HTTPStatusError): - await self.client.get("/endpoint") - mock_request.assert_called_with("GET", "https://api.example.com/endpoint", params={}, headers={}, json=None) + await client.get("/endpoint") + + assert mock_request.call_count == self.retries + 1 + + mock_request.assert_called_with(method="GET", url="https://api.example.com/endpoint", params={}, headers={}) async def test_handle_server_error_response(self): mock_response = httpx.Response(500) mock_response._request = httpx.Request("GET", "https://api.example.com/endpoint") + client = AsyncHttpClient(self.base_url, retries=self.retries) + with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: with self.assertRaises(httpx.HTTPStatusError): - await self.client.get("/endpoint") - mock_request.assert_called_with("GET", "https://api.example.com/endpoint", params={}, headers={}, json=None) + await client.get("/endpoint") + + assert mock_request.call_count == self.retries + 1 + + mock_request.assert_called_with(method="GET", url="https://api.example.com/endpoint", params={}, headers={}) @patch.object(httpx.AsyncClient, 'request') async def test_post_raw_default_pars_with_none_custom_pars_passes(self, mock_request): url = f"{self.base_url}/endpoint" test_def_par = {"default_par": "test"} - await self.client.post_raw("/endpoint", params=test_def_par) + client = AsyncHttpClient(self.base_url, retries=self.retries) + + await client.post_raw("/endpoint", params=test_def_par) + + mock_request.assert_called_once_with(method="POST", url=url, params=test_def_par, headers={}) + + @patch.object(httpx.AsyncClient, 'request') + async def test_post_default_pars_with_none_custom_pars_passes(self, mock_request): + url = f"{self.base_url}/endpoint" + test_def_par = {"default_par": "test"} + + client = AsyncHttpClient(self.base_url, retries=self.retries) + + await client.post("/endpoint", params=test_def_par) + + mock_request.assert_called_once_with(method="POST", url=url, params=test_def_par, headers={}) + + @patch.object(httpx.AsyncClient, 'request') + async def test_post_raw_default_pars_with_custom_pars_passes(self, mock_request): + url = f"{self.base_url}/endpoint" + test_def_par = {"default_par": "test"} + cust_par = {"custom_par": "custom_par_value"} + + client = AsyncHttpClient(self.base_url, retries=self.retries, default_params=test_def_par) + + await client.post_raw("/endpoint", params=cust_par) + + test_cust_def_par = {**test_def_par, **cust_par} + mock_request.assert_called_once_with(method="POST", url=url, params=test_cust_def_par, headers={}) + + @patch.object(httpx.AsyncClient, 'request') + async def test_post_default_pars_with_custom_pars_passes(self, mock_request): + url = f"{self.base_url}/endpoint" + test_def_par = {"default_par": "test"} + cust_par = {"custom_par": "custom_par_value"} + + client = AsyncHttpClient(self.base_url, retries=self.retries, default_params=test_def_par) + + await client.post("/endpoint", params=cust_par) + + test_cust_def_par = {**test_def_par, **cust_par} + mock_request.assert_called_once_with(method="POST", url=url, params=test_cust_def_par, headers={}) + + @patch.object(httpx.AsyncClient, 'request') + async def test_post_raw_default_pars_with_custom_pars_to_None_passes(self, mock_request): + url = f"{self.base_url}/endpoint" + test_def_par = {"default_par": "test"} + cust_par = None + + client = AsyncHttpClient(self.base_url, retries=self.retries, default_params=test_def_par) + + await client.post_raw("/endpoint", params=cust_par) + + # post_raw changes None to empty dict + _cust_par_transformed = {} + test_cust_def_par = {**test_def_par, **_cust_par_transformed} + mock_request.assert_called_once_with(method="POST", url=url, params=test_cust_def_par, headers={}) + + @patch.object(httpx.AsyncClient, 'request') + async def test_post_default_pars_with_custom_pars_to_None_passes(self, mock_request): + url = f"{self.base_url}/endpoint" + test_def_par = {"default_par": "test"} + cust_par = None + + client = AsyncHttpClient(self.base_url, retries=self.retries, default_params=test_def_par) + + await client.post("/endpoint", params=cust_par) + + # post_raw changes None to empty dict + _cust_par_transformed = {} + test_cust_def_par = {**test_def_par, **_cust_par_transformed} + mock_request.assert_called_once_with(method="POST", url=url, params=test_cust_def_par, headers={}) + + @patch.object(httpx.AsyncClient, 'request') + async def test_all_methods_requests_raw_with_custom_pars_passes(self, mock_request): + client = AsyncHttpClient(self.base_url, retries=self.retries) - mock_request.assert_called_once_with( - "POST", - url, - params=test_def_par, - headers={}, - json=None - ) + cust_par = {"custom_par": "custom_par_value"} + for met in ['GET', 'POST', 'PATCH', 'UPDATE', 'PUT']: + await client._request(met, ignore_auth=False, params=cust_par) + mock_request.assert_called_with(method=met, url=self.base_url, params=cust_par, headers={}) if __name__ == "__main__": From 7dcdd61232ec9128dd5a35c3506fa66167c3f86d Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Thu, 25 May 2023 15:15:44 +0200 Subject: [PATCH 12/56] finalize tests --- src/keboola/http_client/async_client.py | 36 +++--- tests/test_async.py | 149 +++++++++++++++++++++--- 2 files changed, 152 insertions(+), 33 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 59291d4..80c2391 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -9,7 +9,7 @@ class AsyncHttpClient: """ An asynchronous HTTP client that simplifies making requests to a specific API. """ - + ALLOWED_METHODS = ['GET', 'POST', 'PATCH', 'UPDATE', 'PUT', 'DELETE'] def __init__( self, base_url: str, @@ -19,6 +19,7 @@ def __init__( retry_status_codes: Optional[List[int]] = None, rate_limit: Optional[int] = None, default_params: Optional[Dict[str, str]] = None, + auth: Optional[tuple] = None, auth_header: Optional[Dict[str, str]] = None, default_headers: Optional[Dict[str, str]] = None, backoff_factor: float = 2.0 @@ -34,6 +35,7 @@ def __init__( retry_status_codes (Optional[List[int]], optional): List of status codes to retry on. Defaults to None. rate_limit (Optional[int], optional): Maximum number of concurrent requests. Defaults to None. default_params (Optional[Dict[str, str]], optional): Default query parameters for each request. + auth (Optional[tuple], optional): Authentication credentials for each request. Defaults to None. auth_header (Optional[Dict[str, str]], optional): Authentication header for each request. Defaults to None. backoff_factor (float, optional): The backoff factor for retries. Defaults to 2.0. """ @@ -44,14 +46,16 @@ def __init__( self.retry_status_codes = retry_status_codes or [] self.rate_limit = rate_limit self.default_params = default_params or {} + self.auth = auth self.auth_header = auth_header or {} self.semaphore = asyncio.Semaphore(rate_limit) if rate_limit else None self.default_headers = default_headers or {} self.backoff_factor = backoff_factor - self.client = httpx.AsyncClient(timeout=self.timeout, verify=self.verify_ssl, headers=self.default_headers) + self.client = httpx.AsyncClient(timeout=self.timeout, verify=self.verify_ssl, headers=self.default_headers, + auth=self.auth) - async def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False): + async def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False) -> str: # build URL Specification url_path = str(endpoint_path).strip() if endpoint_path is not None else '' @@ -80,33 +84,35 @@ async def _request( endpoint: Optional[str] = None, params: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, - json: Optional[Dict[str, Any]] = None, **kwargs ) -> httpx.Response: + is_absolute_path = kwargs.pop('is_absolute_path', False) url = await self._build_url(endpoint, is_absolute_path) all_params = {**self.default_params, **(params or {})} - all_headers = {**self.auth_header, **self.default_headers, **(headers or {})} - request_kwargs = { - "method": method, - "url": url, - "params": all_params, - "headers": all_headers - } + ignore_auth = kwargs.pop('ignore_auth', False) + if ignore_auth: + all_headers = {**self.default_headers, **(headers or {})} + else: + all_headers = {**self.auth_header, **self.default_headers, **(headers or {})} + if self.auth: + kwargs.update({'auth': self.auth}) - if json is not None: - request_kwargs["json"] = json + if all_params: + kwargs.update({'params': all_params}) + if all_headers: + kwargs.update({'headers': all_headers}) for retry_attempt in range(self.retries + 1): try: if self.semaphore: async with self.semaphore: - response = await self.client.request(**request_kwargs) + response = await self.client.request(method, url=url, **kwargs) else: - response = await self.client.request(**request_kwargs) + response = await self.client.request(method, url=url, **kwargs) if response.status_code not in self.retry_status_codes: response.raise_for_status() diff --git a/tests/test_async.py b/tests/test_async.py index d983276..c9459f8 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -2,6 +2,8 @@ from unittest.mock import patch import httpx from keboola.http_client import AsyncHttpClient +from urllib import parse + class TestAsyncHttpClient(unittest.IsolatedAsyncioTestCase): base_url = "https://api.example.com" @@ -17,8 +19,7 @@ async def test_get(self): with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: response = await client.get("/endpoint") self.assertEqual(response, expected_response) - mock_request.assert_called_once_with(method="GET", url="https://api.example.com/endpoint", params={}, - headers={}) + mock_request.assert_called_once_with("GET", url="https://api.example.com/endpoint") async def test_post(self): expected_response = {"message": "Success"} @@ -30,8 +31,8 @@ async def test_post(self): with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: response = await client.post("/endpoint", json={"data": "example"}) self.assertEqual(response, expected_response) - mock_request.assert_called_once_with(method="POST", url="https://api.example.com/endpoint", params={}, - headers={}, json={"data": "example"}) + mock_request.assert_called_once_with("POST", url="https://api.example.com/endpoint", + json={"data": "example"}) async def test_handle_success_response(self): expected_response = {"message": "Success"} @@ -43,8 +44,7 @@ async def test_handle_success_response(self): with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: response = await client.get("/endpoint") self.assertEqual(response, expected_response) - mock_request.assert_called_once_with(method="GET", url="https://api.example.com/endpoint", params={}, - headers={}) + mock_request.assert_called_once_with("GET", url="https://api.example.com/endpoint") async def test_handle_client_error_response(self): mock_response = httpx.Response(404) @@ -58,7 +58,7 @@ async def test_handle_client_error_response(self): assert mock_request.call_count == self.retries + 1 - mock_request.assert_called_with(method="GET", url="https://api.example.com/endpoint", params={}, headers={}) + mock_request.assert_called_with("GET", url="https://api.example.com/endpoint") async def test_handle_server_error_response(self): mock_response = httpx.Response(500) @@ -72,7 +72,7 @@ async def test_handle_server_error_response(self): assert mock_request.call_count == self.retries + 1 - mock_request.assert_called_with(method="GET", url="https://api.example.com/endpoint", params={}, headers={}) + mock_request.assert_called_with("GET", url="https://api.example.com/endpoint") @patch.object(httpx.AsyncClient, 'request') async def test_post_raw_default_pars_with_none_custom_pars_passes(self, mock_request): @@ -83,7 +83,7 @@ async def test_post_raw_default_pars_with_none_custom_pars_passes(self, mock_req await client.post_raw("/endpoint", params=test_def_par) - mock_request.assert_called_once_with(method="POST", url=url, params=test_def_par, headers={}) + mock_request.assert_called_once_with("POST", url=url, params=test_def_par) @patch.object(httpx.AsyncClient, 'request') async def test_post_default_pars_with_none_custom_pars_passes(self, mock_request): @@ -94,7 +94,7 @@ async def test_post_default_pars_with_none_custom_pars_passes(self, mock_request await client.post("/endpoint", params=test_def_par) - mock_request.assert_called_once_with(method="POST", url=url, params=test_def_par, headers={}) + mock_request.assert_called_once_with("POST", url=url, params=test_def_par) @patch.object(httpx.AsyncClient, 'request') async def test_post_raw_default_pars_with_custom_pars_passes(self, mock_request): @@ -107,7 +107,7 @@ async def test_post_raw_default_pars_with_custom_pars_passes(self, mock_request) await client.post_raw("/endpoint", params=cust_par) test_cust_def_par = {**test_def_par, **cust_par} - mock_request.assert_called_once_with(method="POST", url=url, params=test_cust_def_par, headers={}) + mock_request.assert_called_once_with("POST", url=url, params=test_cust_def_par) @patch.object(httpx.AsyncClient, 'request') async def test_post_default_pars_with_custom_pars_passes(self, mock_request): @@ -120,7 +120,7 @@ async def test_post_default_pars_with_custom_pars_passes(self, mock_request): await client.post("/endpoint", params=cust_par) test_cust_def_par = {**test_def_par, **cust_par} - mock_request.assert_called_once_with(method="POST", url=url, params=test_cust_def_par, headers={}) + mock_request.assert_called_once_with("POST", url=url, params=test_cust_def_par) @patch.object(httpx.AsyncClient, 'request') async def test_post_raw_default_pars_with_custom_pars_to_None_passes(self, mock_request): @@ -135,7 +135,7 @@ async def test_post_raw_default_pars_with_custom_pars_to_None_passes(self, mock_ # post_raw changes None to empty dict _cust_par_transformed = {} test_cust_def_par = {**test_def_par, **_cust_par_transformed} - mock_request.assert_called_once_with(method="POST", url=url, params=test_cust_def_par, headers={}) + mock_request.assert_called_once_with("POST", url=url, params=test_cust_def_par) @patch.object(httpx.AsyncClient, 'request') async def test_post_default_pars_with_custom_pars_to_None_passes(self, mock_request): @@ -150,17 +150,130 @@ async def test_post_default_pars_with_custom_pars_to_None_passes(self, mock_requ # post_raw changes None to empty dict _cust_par_transformed = {} test_cust_def_par = {**test_def_par, **_cust_par_transformed} - mock_request.assert_called_once_with(method="POST", url=url, params=test_cust_def_par, headers={}) + mock_request.assert_called_once_with("POST", url=url, params=test_cust_def_par) @patch.object(httpx.AsyncClient, 'request') async def test_all_methods_requests_raw_with_custom_pars_passes(self, mock_request): - client = AsyncHttpClient(self.base_url, retries=self.retries) + client = AsyncHttpClient(self.base_url) cust_par = {"custom_par": "custom_par_value"} - for met in ['GET', 'POST', 'PATCH', 'UPDATE', 'PUT']: - await client._request(met, ignore_auth=False, params=cust_par) - mock_request.assert_called_with(method=met, url=self.base_url, params=cust_par, headers={}) + for m in client.ALLOWED_METHODS: + await client._request(m, ignore_auth=False, params=cust_par) + mock_request.assert_called_with(m, url=self.base_url+"/", params=cust_par) + + @patch.object(httpx.AsyncClient, 'request') + async def test_all_methods_skip_auth(self, mock_request): + client = AsyncHttpClient(self.base_url, auth=("my_user", "password123")) + + for m in ['GET', 'POST', 'PATCH', 'UPDATE', 'PUT', 'DELETE']: + await client._request(m, ignore_auth=True) + mock_request.assert_called_with(m, url=self.base_url+"/") + + @patch.object(httpx.AsyncClient, 'request') + async def test_request_skip_auth_header(self, mock_request): + def_header = {"def_header": "test"} + client = AsyncHttpClient('http://example.com', default_headers=def_header, + auth_header={"Authorization": "test"}) + + await client._request('POST', 'abc', ignore_auth=True) + mock_request.assert_called_with('POST', url="http://example.com/abc", headers=def_header) + + @patch.object(httpx.AsyncClient, 'request') + async def test_request_auth(self, mock_request): + def_header = {"def_header": "test"} + auth = ("my_user", "password123") + client = AsyncHttpClient(self.base_url, auth=auth, default_headers=def_header) + + await client._request('POST', 'abc') + mock_request.assert_called_with('POST', url=self.base_url+"/abc", headers=def_header, + auth=auth) + + @patch.object(httpx.AsyncClient, 'request') + async def test_all_methods(self, mock_request): + client = AsyncHttpClient(self.base_url, default_headers={'header1': 'headerval'}, + auth_header={'api_token': 'abdc1234'}) + + target_url = f'{self.base_url}/abc' + + for m in client.ALLOWED_METHODS: + await client._request(m, 'abc', params={'exclude': 'componentDetails'}, headers={'abc': '123'}, + data={'attr1': 'val1'}) + mock_request.assert_called_with(m, url=target_url, + params={'exclude': 'componentDetails'}, + headers={'api_token': 'abdc1234', 'header1': 'headerval', 'abc': '123'}, + data={'attr1': 'val1'}) + + @patch.object(httpx.AsyncClient, 'request') + async def test_all_methods_requests_raw_with_is_absolute_path_true(self, mock_request): + def_header = {"def_header": "test"} + client = AsyncHttpClient(self.base_url, default_headers=def_header) + + for m in client.ALLOWED_METHODS: + await client._request(m, 'http://example2.com/v1/', is_absolute_path=True) + mock_request.assert_called_with(m, url='http://example2.com/v1/', headers=def_header) + + @patch.object(httpx.AsyncClient, 'request') + async def test_all_methods_requests_raw_with_is_absolute_path_false(self, mock_request): + def_header = {"def_header": "test"} + client = AsyncHttpClient(self.base_url, default_headers=def_header) + + for m in client.ALLOWED_METHODS: + await client._request(m, 'cars') + mock_request.assert_called_with(m, url=self.base_url+"/cars", headers=def_header) + + @patch.object(httpx.AsyncClient, 'request') + async def test_all_methods_kwargs(self, mock_request): + client = AsyncHttpClient(self.base_url) + + for m in client.ALLOWED_METHODS: + await client._request(m, 'cars', data={'data': '123'}, cert='/path/to/cert', files={'a': '/path/to/file'}, + params={'par1': 'val1'}) + + mock_request.assert_called_with(m, url=self.base_url+"/cars", data={'data': '123'}, + cert='/path/to/cert', files={'a': '/path/to/file'}, + params={'par1': 'val1'}) + + async def test_build_url_rel_path(self): + url = 'https://example.com/' + cl = AsyncHttpClient(url) + expected_url = 'https://example.com/storage' + actual_url = await cl._build_url('storage') + self.assertEqual(expected_url, actual_url) + + async def test_build_url_abs_path(self): + url = 'https://example.com/' + cl = AsyncHttpClient(url) + expected_url = 'https://example2.com/storage' + actual_url = await cl._build_url('https://example2.com/storage', True) + self.assertEqual(expected_url, actual_url) + + async def test_build_url_empty_endpoint_path_leads_to_base_url(self): + url = 'https://example.com/' + cl = AsyncHttpClient(url) + expected_url = url + + actual_url = await cl._build_url() + self.assertEqual(expected_url, actual_url) + + actual_url = await cl._build_url('') + self.assertEqual(expected_url, actual_url) + + actual_url = await cl._build_url(None) + self.assertEqual(expected_url, actual_url) + + actual_url = await cl._build_url('', is_absolute_path=True) + self.assertEqual(expected_url, actual_url) + + actual_url = await cl._build_url(None, is_absolute_path=True) + self.assertEqual(expected_url, actual_url) + + async def test_build_url_base_url_appends_slash(self): + url = 'https://example.com' + cl = AsyncHttpClient(url) + expected_base_url = 'https://example.com/' + + self.assertEqual(expected_base_url, cl.base_url) if __name__ == "__main__": From 35033cc12f0e14e72d8b5b9df6be74f4d58420f1 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Thu, 25 May 2023 16:30:55 +0200 Subject: [PATCH 13/56] add Storage API example to docs --- README.md | 37 +++++++++++++ ...sync_poekapi_large.py => async_poekapi.py} | 0 docs/examples/async_pokeapi.py | 55 ------------------- docs/examples/async_storage_client.py | 29 ++++++++++ docs/examples/{sync_poekapi.py => poekapi.py} | 0 docs/examples/storage_client.py | 20 +++++++ 6 files changed, 86 insertions(+), 55 deletions(-) rename docs/examples/{async_poekapi_large.py => async_poekapi.py} (100%) delete mode 100644 docs/examples/async_pokeapi.py create mode 100644 docs/examples/async_storage_client.py rename docs/examples/{sync_poekapi.py => poekapi.py} (100%) create mode 100644 docs/examples/storage_client.py diff --git a/README.md b/README.md index 2963949..0e8aa67 100644 --- a/README.md +++ b/README.md @@ -237,3 +237,40 @@ asyncio.run(main()) The AsyncHttpClient provides similar initialization and request methods as the HttpClient. The request methods return awaitable coroutines that can be awaited in an asynchronous context. + +#### Building HTTP client based on AsyncHttpClient Example +This example demonstrates the default use of the HTTPClient as a base for REST API clients. + +```python +import asyncio +from keboola.http_client import AsyncHttpClient + +BASE_URL = 'https://connection.keboola.com/v2/storage' +MAX_RETRIES = 3 + +class KBCStorageClient(AsyncHttpClient): + + def __init__(self, storage_token): + AsyncHttpClient.__init__( + self, + base_url=BASE_URL, + retries=MAX_RETRIES, + backoff_factor=0.3, + retry_status_codes=[429, 500, 502, 504], + auth_header={"X-StorageApi-Token": storage_token} + ) + + async def get_files(self, show_expired=False): + params = {"showExpired": show_expired} + response = await self.get('tables', params=params, timeout=5) + return response + +async def main(): + cl = KBCStorageClient("my_token") + files = await cl.get_files(show_expired=False) + print(files) + +asyncio.run(main()) +``` +**Note:** Since there are no parallel requests being made, you won't notice any speedup for this use case. +For an example where you can see the speedup thanks to async requests, you can view the pokeapi.py in docs/examples. diff --git a/docs/examples/async_poekapi_large.py b/docs/examples/async_poekapi.py similarity index 100% rename from docs/examples/async_poekapi_large.py rename to docs/examples/async_poekapi.py diff --git a/docs/examples/async_pokeapi.py b/docs/examples/async_pokeapi.py deleted file mode 100644 index 1cb74b4..0000000 --- a/docs/examples/async_pokeapi.py +++ /dev/null @@ -1,55 +0,0 @@ -import time -import asyncio -from keboola.http_client import AsyncHttpClient -import csv -import httpx - -async def fetch_pokemon_details_async(client: AsyncHttpClient, endpoint: str): - r = await client.get(endpoint) - return r - -async def main_async(): - base_url = "https://pokeapi.co/api/v2/pokemon/" - start_time = time.time() - client = AsyncHttpClient(base_url=base_url) - - async def fetch_pokemon(client, poke_id): - try: - r = await fetch_pokemon_details_async(client, str(poke_id)) - return r - except httpx.HTTPStatusError as e: - if e.response.status_code == 404: - return None - else: - raise - - async with client as c: - pokemon_details = [] - poke_id = 1 - - while True: - details = await fetch_pokemon(c, poke_id) - if details is None: - break - pokemon_details.append(details) - poke_id += 1 - - # Save details to CSV - filename = "pokemon_details.csv" - fieldnames = ["name", "height", "weight"] # Define the fields you want to store - - with open(filename, "w", newline="") as csvfile: - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - writer.writeheader() - for details in pokemon_details: - writer.writerow({ - "name": details["name"], - "height": details["height"], - "weight": details["weight"] - }) - - end_time = time.time() - print(f"Async: Fetched details for {len(pokemon_details)} Pokémon in {end_time - start_time:.2f} seconds.") - -if __name__ == "__main__": - asyncio.run(main_async()) diff --git a/docs/examples/async_storage_client.py b/docs/examples/async_storage_client.py new file mode 100644 index 0000000..3200242 --- /dev/null +++ b/docs/examples/async_storage_client.py @@ -0,0 +1,29 @@ +import asyncio +from keboola.http_client import AsyncHttpClient + +BASE_URL = 'https://connection.keboola.com/v2/storage' +MAX_RETRIES = 3 + +class KBCStorageClient(AsyncHttpClient): + + def __init__(self, storage_token): + AsyncHttpClient.__init__( + self, + base_url=BASE_URL, + retries=MAX_RETRIES, + backoff_factor=0.3, + retry_status_codes=[429, 500, 502, 504], + auth_header={"X-StorageApi-Token": storage_token} + ) + + async def get_files(self, show_expired=False): + params = {"showExpired": show_expired} + response = await self.get('tables', params=params, timeout=5) + return response + +async def main(): + cl = KBCStorageClient("my_token") + files = await cl.get_files(show_expired=False) + print(files) + +asyncio.run(main()) diff --git a/docs/examples/sync_poekapi.py b/docs/examples/poekapi.py similarity index 100% rename from docs/examples/sync_poekapi.py rename to docs/examples/poekapi.py diff --git a/docs/examples/storage_client.py b/docs/examples/storage_client.py new file mode 100644 index 0000000..d82dbd1 --- /dev/null +++ b/docs/examples/storage_client.py @@ -0,0 +1,20 @@ +from keboola.http_client import HttpClient + +BASE_URL = 'https://connection.keboola.com/v2/storage' +MAX_RETRIES = 3 + + +class KBCStorageClient(HttpClient): + + def __init__(self, storage_token): + HttpClient.__init__(self, base_url=BASE_URL, max_retries=MAX_RETRIES, backoff_factor=0.3, + status_forcelist=(429, 500, 502, 504), + default_http_header={"X-StorageApi-Token": storage_token}) + + def get_files(self, show_expired=None): + params = {"include": show_expired} + return self.get('tables', params=params, timeout=5) + +cl = KBCStorageClient("my_token") + +print(cl.get_files()) \ No newline at end of file From 7406d34e85e71d9723b9875c7f9654d57c4af222 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Thu, 25 May 2023 16:41:51 +0200 Subject: [PATCH 14/56] update backoff calculation --- src/keboola/http_client/async_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 80c2391..3788981 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -122,8 +122,8 @@ async def _request( except httpx.HTTPError: if retry_attempt == self.retries: raise - backoff_factor = 2 ** retry_attempt - await asyncio.sleep(random.uniform(0, backoff_factor)) + backoff = self.backoff_factor ** retry_attempt + await asyncio.sleep(backoff) async def get(self, endpoint: Optional[str] = None, **kwargs) -> Any: response = await self.get_raw(endpoint, **kwargs) From c294a6a4fee17ceb6fca5fac2bb347034393b9b1 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Thu, 25 May 2023 16:46:25 +0200 Subject: [PATCH 15/56] remove unused imports/installs --- .github/workflows/deploy_to_test.yml | 2 +- src/keboola/http_client/async_client.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/deploy_to_test.yml b/.github/workflows/deploy_to_test.yml index 4b18403..72f59c8 100644 --- a/.github/workflows/deploy_to_test.yml +++ b/.github/workflows/deploy_to_test.yml @@ -20,7 +20,7 @@ jobs: run: | python -m pip install --upgrade pip pip install setuptools wheel twine - pip install flake8 pytest pytest-asyncio # Add pytest-asyncio here + pip install flake8 pytest pytest-asyncio pip install -r requirements.txt - name: Lint with flake8 run: | diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 3788981..61193a1 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -1,4 +1,3 @@ -import random import httpx import asyncio from typing import Optional, Dict, Any, List From 8033dc9d65b18a6743a657c11d0acfd0871df6ff Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Thu, 25 May 2023 16:55:35 +0200 Subject: [PATCH 16/56] remove more unused imports/installs --- .github/workflows/deploy_to_test.yml | 2 +- tests/test_async.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/deploy_to_test.yml b/.github/workflows/deploy_to_test.yml index 72f59c8..0a61c7f 100644 --- a/.github/workflows/deploy_to_test.yml +++ b/.github/workflows/deploy_to_test.yml @@ -20,7 +20,7 @@ jobs: run: | python -m pip install --upgrade pip pip install setuptools wheel twine - pip install flake8 pytest pytest-asyncio + pip install flake8 pytest pip install -r requirements.txt - name: Lint with flake8 run: | diff --git a/tests/test_async.py b/tests/test_async.py index c9459f8..b582117 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -2,7 +2,6 @@ from unittest.mock import patch import httpx from keboola.http_client import AsyncHttpClient -from urllib import parse class TestAsyncHttpClient(unittest.IsolatedAsyncioTestCase): From 5c29ef20494ae553fa85f3443f11ad377ee61841 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Fri, 26 May 2023 09:32:46 +0200 Subject: [PATCH 17/56] add update_auth_header method + tests --- src/keboola/http_client/async_client.py | 19 +++++++++++++++++-- tests/test_async.py | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 61193a1..bd0cd5a 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -46,7 +46,7 @@ def __init__( self.rate_limit = rate_limit self.default_params = default_params or {} self.auth = auth - self.auth_header = auth_header or {} + self._auth_header = auth_header or {} self.semaphore = asyncio.Semaphore(rate_limit) if rate_limit else None self.default_headers = default_headers or {} self.backoff_factor = backoff_factor @@ -67,6 +67,21 @@ async def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path return url + async def update_auth_header(self, updated_header: Dict, overwrite: bool = False): + """ + Updates the default auth header by providing new values. + + Args: + updated_header: An updated header which will be used to update the current header. + overwrite: If `False`, the existing header will be updated with new header. If `True`, the new header will + overwrite (replace) the current authentication header. + """ + + if overwrite is False: + self._auth_header.update(updated_header) + else: + self._auth_header = updated_header + async def __aenter__(self): await self.client.__aenter__() return self @@ -96,7 +111,7 @@ async def _request( if ignore_auth: all_headers = {**self.default_headers, **(headers or {})} else: - all_headers = {**self.auth_header, **self.default_headers, **(headers or {})} + all_headers = {**self._auth_header, **self.default_headers, **(headers or {})} if self.auth: kwargs.update({'auth': self.auth}) diff --git a/tests/test_async.py b/tests/test_async.py index b582117..927d7ef 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -274,6 +274,26 @@ async def test_build_url_base_url_appends_slash(self): self.assertEqual(expected_base_url, cl.base_url) + async def test_update_auth_header_None(self): + existing_header = None + new_header = {'api_token': 'token_value'} + + cl = AsyncHttpClient('https://example.com', auth_header=existing_header) + await cl.update_auth_header(new_header, overwrite=False) + self.assertDictEqual(cl._auth_header, new_header) + + new_header_2 = {'password': '123'} + await cl.update_auth_header(new_header_2, overwrite=True) + self.assertDictEqual(cl._auth_header, new_header_2) + + async def test_update_existing_auth_header(self): + existing_header = {'authorization': 'value'} + new_header = {'api_token': 'token_value'} + + cl = AsyncHttpClient('https://example.com', auth_header=existing_header) + await cl.update_auth_header(new_header, overwrite=False) + self.assertDictEqual(cl._auth_header, {**existing_header, **new_header}) + if __name__ == "__main__": unittest.main() From 3c547553815b7676a3c98fdfb14686046732c5e0 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Wed, 31 May 2023 13:57:38 +0200 Subject: [PATCH 18/56] Update async_storage_client.py --- docs/examples/async_storage_client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/examples/async_storage_client.py b/docs/examples/async_storage_client.py index 3200242..b1ca73f 100644 --- a/docs/examples/async_storage_client.py +++ b/docs/examples/async_storage_client.py @@ -7,8 +7,7 @@ class KBCStorageClient(AsyncHttpClient): def __init__(self, storage_token): - AsyncHttpClient.__init__( - self, + super().__init__( base_url=BASE_URL, retries=MAX_RETRIES, backoff_factor=0.3, From fedba9c44967660fcce22a65da8138ccd035cc61 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Wed, 31 May 2023 14:29:56 +0200 Subject: [PATCH 19/56] add correct typehints --- src/keboola/http_client/async_client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index bd0cd5a..bbfaa43 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -139,35 +139,35 @@ async def _request( backoff = self.backoff_factor ** retry_attempt await asyncio.sleep(backoff) - async def get(self, endpoint: Optional[str] = None, **kwargs) -> Any: + async def get(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, Any]: response = await self.get_raw(endpoint, **kwargs) return response.json() async def get_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: return await self._request("GET", endpoint, **kwargs) - async def post(self, endpoint: Optional[str] = None, **kwargs) -> Any: + async def post(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, Any]: response = await self.post_raw(endpoint, **kwargs) return response.json() async def post_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: return await self._request("POST", endpoint, **kwargs) - async def put(self, endpoint: Optional[str] = None, **kwargs) -> Any: + async def put(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, Any]: response = await self.put_raw(endpoint, **kwargs) return response.json() async def put_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: return await self._request("PUT", endpoint, **kwargs) - async def patch(self, endpoint: Optional[str] = None, **kwargs) -> Any: + async def patch(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, Any]: response = await self.patch_raw(endpoint, **kwargs) return response.json() async def patch_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: return await self._request("PATCH", endpoint, **kwargs) - async def delete(self, endpoint: Optional[str] = None, **kwargs) -> Any: + async def delete(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, Any]: response = await self.delete_raw(endpoint, **kwargs) return response.json() From b387e31b587f52c899803012097a513048123723 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Thu, 1 Jun 2023 16:12:42 +0200 Subject: [PATCH 20/56] enable rate limiting using aiolimiter --- docs/examples/async_poekapi.py | 2 +- requirements.txt | 3 ++- src/keboola/http_client/async_client.py | 17 +++++++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/examples/async_poekapi.py b/docs/examples/async_poekapi.py index 188340c..cb653b6 100644 --- a/docs/examples/async_poekapi.py +++ b/docs/examples/async_poekapi.py @@ -11,7 +11,7 @@ async def fetch_pokemon_details_async(client: AsyncHttpClient, endpoint: str): async def main_async(): base_url = "https://pokeapi.co/api/v2/pokemon/" start_time = time.time() - client = AsyncHttpClient(base_url=base_url) + client = AsyncHttpClient(base_url=base_url, max_requests_per_second=10) async def fetch_pokemon(client, poke_id): try: diff --git a/requirements.txt b/requirements.txt index a0c12bb..2277049 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests -httpx==0.24.0 \ No newline at end of file +httpx==0.24.0 +aiolimiter==1.1.0 \ No newline at end of file diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index bbfaa43..4bc7827 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -2,6 +2,7 @@ import asyncio from typing import Optional, Dict, Any, List from urllib.parse import urljoin +from aiolimiter import AsyncLimiter class AsyncHttpClient: @@ -16,7 +17,7 @@ def __init__( timeout: Optional[float] = None, verify_ssl: bool = True, retry_status_codes: Optional[List[int]] = None, - rate_limit: Optional[int] = None, + max_requests_per_second: Optional[int] = None, default_params: Optional[Dict[str, str]] = None, auth: Optional[tuple] = None, auth_header: Optional[Dict[str, str]] = None, @@ -32,7 +33,7 @@ def __init__( timeout (Optional[float], optional): The request timeout in seconds. Defaults to None. verify_ssl (bool, optional): Enable or disable SSL verification. Defaults to True. retry_status_codes (Optional[List[int]], optional): List of status codes to retry on. Defaults to None. - rate_limit (Optional[int], optional): Maximum number of concurrent requests. Defaults to None. + max_requests_per_second (Optional[int], optional): Maximum number of requests per second. Defaults to None. default_params (Optional[Dict[str, str]], optional): Default query parameters for each request. auth (Optional[tuple], optional): Authentication credentials for each request. Defaults to None. auth_header (Optional[Dict[str, str]], optional): Authentication header for each request. Defaults to None. @@ -43,11 +44,15 @@ def __init__( self.timeout = httpx.Timeout(timeout) if timeout else None self.verify_ssl = verify_ssl self.retry_status_codes = retry_status_codes or [] - self.rate_limit = rate_limit self.default_params = default_params or {} self.auth = auth self._auth_header = auth_header or {} - self.semaphore = asyncio.Semaphore(rate_limit) if rate_limit else None + + self.limiter = None + if max_requests_per_second: + one_reqeust_per_second_amount = float(1/int(max_requests_per_second)) + self.limiter = AsyncLimiter(1, one_reqeust_per_second_amount) + self.default_headers = default_headers or {} self.backoff_factor = backoff_factor @@ -122,8 +127,8 @@ async def _request( for retry_attempt in range(self.retries + 1): try: - if self.semaphore: - async with self.semaphore: + if self.limiter: + async with self.limiter: response = await self.client.request(method, url=url, **kwargs) else: response = await self.client.request(method, url=url, **kwargs) From 56bb576b8ee19ffb4a28f20dc3333c04410e09fb Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Fri, 2 Jun 2023 11:52:41 +0200 Subject: [PATCH 21/56] add process_multiple method and example --- docs/examples/async_poekapi.py | 66 ------------------- docs/examples/poekapi.py | 46 ------------- docs/examples/poekapi_async.py | 61 +++++++++++++++++ docs/examples/pokeapi_process_multiple.py | 44 +++++++++++++ ...rage_client.py => storage_client_async.py} | 0 src/keboola/http_client/async_client.py | 43 ++++++++++++ 6 files changed, 148 insertions(+), 112 deletions(-) delete mode 100644 docs/examples/async_poekapi.py delete mode 100644 docs/examples/poekapi.py create mode 100644 docs/examples/poekapi_async.py create mode 100644 docs/examples/pokeapi_process_multiple.py rename docs/examples/{async_storage_client.py => storage_client_async.py} (100%) diff --git a/docs/examples/async_poekapi.py b/docs/examples/async_poekapi.py deleted file mode 100644 index cb653b6..0000000 --- a/docs/examples/async_poekapi.py +++ /dev/null @@ -1,66 +0,0 @@ -import time -import asyncio -from keboola.http_client import AsyncHttpClient -import csv -import httpx - -async def fetch_pokemon_details_async(client: AsyncHttpClient, endpoint: str): - r = await client.get(endpoint) - return r - -async def main_async(): - base_url = "https://pokeapi.co/api/v2/pokemon/" - start_time = time.time() - client = AsyncHttpClient(base_url=base_url, max_requests_per_second=10) - - async def fetch_pokemon(client, poke_id): - try: - r = await fetch_pokemon_details_async(client, str(poke_id)) - return r - except httpx.HTTPStatusError as e: - if e.response.status_code == 404: - return None - else: - raise - - async with client as c: - poke_id = 1 - - # Check if the file exists and write the header if it doesn't - filename = "pokemon_details.csv" - fieldnames = ["name", "height", "weight"] - - try: - with open(filename, "x", newline="") as csvfile: - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - writer.writeheader() - except FileExistsError: - pass - - while True: - details = await fetch_pokemon(c, poke_id) - if details is None: - break - - await save_to_csv(details) - - poke_id += 1 - - end_time = time.time() - print(f"Async: Fetched details for {poke_id - 1} Pokémon in {end_time - start_time:.2f} seconds.") - -async def save_to_csv(details): - filename = "pokemon_details.csv" - fieldnames = ["name", "height", "weight"] - - with open(filename, "a", newline="") as csvfile: - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - - writer.writerow({ - "name": details["name"], - "height": details["height"], - "weight": details["weight"] - }) - -if __name__ == "__main__": - asyncio.run(main_async()) diff --git a/docs/examples/poekapi.py b/docs/examples/poekapi.py deleted file mode 100644 index 1f58487..0000000 --- a/docs/examples/poekapi.py +++ /dev/null @@ -1,46 +0,0 @@ -import requests -import time -import csv - -def fetch_pokemon_details_sync(url: str): - r = requests.get(url) - r.raise_for_status() - return r.json() - -def main_sync(): - base_url = "https://pokeapi.co/api/v2/pokemon/" - start_time = time.time() - pokemon_details = [] - poke_id = 1 - - while True: - url = f"{base_url}{poke_id}" - try: - details = fetch_pokemon_details_sync(url) - pokemon_details.append(details) - poke_id += 1 - except requests.exceptions.HTTPError as e: - if e.response.status_code == 404: - break - else: - raise - - # Save details to CSV - filename = "pokemon_details.csv" - fieldnames = ["name", "height", "weight"] # Define the fields you want to store - - with open(filename, "w", newline="") as csvfile: - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - writer.writeheader() - for details in pokemon_details: - writer.writerow({ - "name": details["name"], - "height": details["height"], - "weight": details["weight"] - }) - - end_time = time.time() - print(f"Fetched details for {len(pokemon_details)} Pokémon in {end_time - start_time:.2f} seconds.") - -if __name__ == "__main__": - main_sync() diff --git a/docs/examples/poekapi_async.py b/docs/examples/poekapi_async.py new file mode 100644 index 0000000..915cac0 --- /dev/null +++ b/docs/examples/poekapi_async.py @@ -0,0 +1,61 @@ +import time +import asyncio +from keboola.http_client import AsyncHttpClient +import csv +import httpx +import os + + +async def fetch_pokemon(client, poke_id): + try: + r = await client.get(str(poke_id)) + return r + except httpx.HTTPStatusError as e: + if e.response.status_code == 404: + return None + else: + raise + + +async def save_to_csv(details): + filename = "pokemon_details.csv" + fieldnames = ["name", "height", "weight"] + + file_exists = os.path.isfile(filename) + mode = "a" if file_exists else "w" + + with open(filename, mode, newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + + if not file_exists: + writer.writeheader() + + writer.writerow({ + "name": details["name"], + "height": details["height"], + "weight": details["weight"] + }) + + +async def main_async(): + base_url = "https://pokeapi.co/api/v2/pokemon/" + start_time = time.time() + + async with AsyncHttpClient(base_url=base_url, max_requests_per_second=20) as c: + poke_id = 1 + + while True: + details = await fetch_pokemon(c, poke_id) + if details is None: + break + + await save_to_csv(details) + + poke_id += 1 + + end_time = time.time() + print(f"Async: Fetched details for {poke_id - 1} Pokémon in {end_time - start_time:.2f} seconds.") + + +if __name__ == "__main__": + asyncio.run(main_async()) diff --git a/docs/examples/pokeapi_process_multiple.py b/docs/examples/pokeapi_process_multiple.py new file mode 100644 index 0000000..1681877 --- /dev/null +++ b/docs/examples/pokeapi_process_multiple.py @@ -0,0 +1,44 @@ +import asyncio +import csv +import time +from typing import List + +from keboola.http_client import AsyncHttpClient + + +def generate_jobs(nr_of_jobs): + return [{'method': 'GET', 'endpoint': str(endpoint)} for endpoint in range(1, nr_of_jobs+1)] + +def save_to_csv(results: List[dict]): + filename = "pokemon_details.csv" + fieldnames = ["name", "height", "weight"] # Define the fields you want to store + + with open(filename, "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + for result in results: + writer.writerow({ + "name": result["name"], + "height": result["height"], + "weight": result["weight"] + }) + +async def main_async(): + base_url = "https://pokeapi.co/api/v2/pokemon/" + start_time = time.time() + + client = AsyncHttpClient(base_url=base_url, max_requests_per_second=20) + + jobs = generate_jobs(1000) + + results = await client.process_multiple(jobs) + await client.close() + + end_time = time.time() + print(f"Fetched details for {len(results)} Pokémon in {end_time - start_time:.2f} seconds.") + + save_to_csv(results) + + +if __name__ == "__main__": + asyncio.run(main_async()) diff --git a/docs/examples/async_storage_client.py b/docs/examples/storage_client_async.py similarity index 100% rename from docs/examples/async_storage_client.py rename to docs/examples/storage_client_async.py diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 4bc7827..d84bfcc 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -178,3 +178,46 @@ async def delete(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, An async def delete_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: return await self._request("DELETE", endpoint, **kwargs) + + async def process_multiple(self, jobs: List[Dict[str, Any]]): + tasks = [] + + for job in jobs: + method = job['method'] + endpoint = job['endpoint'] + params = job.get('params') + headers = job.get('headers') + raw = job.get('raw', False) + + if method == 'GET': + if raw: + task = self.get_raw(endpoint, params=params, headers=headers) + else: + task = self.get(endpoint, params=params, headers=headers) + elif method == 'POST': + if raw: + task = self.post_raw(endpoint, params=params, headers=headers) + else: + task = self.post(endpoint, params=params, headers=headers) + elif method == 'PUT': + if raw: + task = self.put_raw(endpoint, params=params, headers=headers) + else: + task = self.put(endpoint, params=params, headers=headers) + elif method == 'PATCH': + if raw: + task = self.patch_raw(endpoint, params=params, headers=headers) + else: + task = self.patch(endpoint, params=params, headers=headers) + elif method == 'DELETE': + if raw: + task = self.delete_raw(endpoint, params=params, headers=headers) + else: + task = self.delete(endpoint, params=params, headers=headers) + else: + raise ValueError(f"Unsupported method: {method}") + + tasks.append(task) + + responses = await asyncio.gather(*tasks) + return responses From 865a7113b786a7f38d190cbceccdffde7a3719ad Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Wed, 14 Jun 2023 11:34:29 +0200 Subject: [PATCH 22/56] refactor retry strategy --- .gitignore | 5 ++++- src/keboola/http_client/async_client.py | 9 +++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 6b1153c..28d81a5 100644 --- a/.gitignore +++ b/.gitignore @@ -139,4 +139,7 @@ dmypy.json .idea/ # MacOS files -.DS_Store \ No newline at end of file +.DS_Store + +# Local test script +/docs/examples/test.py diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index d84bfcc..1869dbe 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -125,6 +125,8 @@ async def _request( if all_headers: kwargs.update({'headers': all_headers}) + response = None + for retry_attempt in range(self.retries + 1): try: if self.limiter: @@ -133,12 +135,15 @@ async def _request( else: response = await self.client.request(method, url=url, **kwargs) - if response.status_code not in self.retry_status_codes: - response.raise_for_status() + response.raise_for_status() return response except httpx.HTTPError: + if response: + if response.status_code not in self.retry_status_codes: + raise + if retry_attempt == self.retries: raise backoff = self.backoff_factor ** retry_attempt From 7270fa1e3f6f44e556afce9febe3db34824af638 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Fri, 8 Sep 2023 00:13:27 +0200 Subject: [PATCH 23/56] fix tests --- src/keboola/http_client/async_client.py | 8 ++++---- tests/test_async.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 1869dbe..7136c76 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -43,7 +43,7 @@ def __init__( self.retries = retries self.timeout = httpx.Timeout(timeout) if timeout else None self.verify_ssl = verify_ssl - self.retry_status_codes = retry_status_codes or [] + self.retry_status_codes = retry_status_codes or [429, 500, 502, 504] self.default_params = default_params or {} self.auth = auth self._auth_header = auth_header or {} @@ -65,10 +65,10 @@ async def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path if not url_path: url = self.base_url - elif not is_absolute_path: - url = urljoin(self.base_url, endpoint_path) - else: + elif is_absolute_path: url = endpoint_path + else: + url = urljoin(self.base_url, endpoint_path) return url diff --git a/tests/test_async.py b/tests/test_async.py index 927d7ef..5c9618a 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -49,7 +49,7 @@ async def test_handle_client_error_response(self): mock_response = httpx.Response(404) mock_response._request = httpx.Request("GET", "https://api.example.com/endpoint") - client = AsyncHttpClient(self.base_url, retries=self.retries) + client = AsyncHttpClient(self.base_url, retries=self.retries, retry_status_codes=[404]) with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: with self.assertRaises(httpx.HTTPStatusError): @@ -63,7 +63,7 @@ async def test_handle_server_error_response(self): mock_response = httpx.Response(500) mock_response._request = httpx.Request("GET", "https://api.example.com/endpoint") - client = AsyncHttpClient(self.base_url, retries=self.retries) + client = AsyncHttpClient(self.base_url, retries=self.retries, retry_status_codes=[500]) with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: with self.assertRaises(httpx.HTTPStatusError): From 677b0f5b41e127ced6db6b18496196971cf2272b Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Fri, 27 Oct 2023 15:49:26 +0200 Subject: [PATCH 24/56] add workaround to disable INFO msgs coming from httpx library --- requirements.txt | 2 +- src/keboola/http_client/async_client.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2277049..d8e5962 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ requests httpx==0.24.0 -aiolimiter==1.1.0 \ No newline at end of file +aiolimiter==1.1.0 diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 7136c76..bdd4942 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -3,6 +3,7 @@ from typing import Optional, Dict, Any, List from urllib.parse import urljoin from aiolimiter import AsyncLimiter +import logging class AsyncHttpClient: @@ -22,7 +23,8 @@ def __init__( auth: Optional[tuple] = None, auth_header: Optional[Dict[str, str]] = None, default_headers: Optional[Dict[str, str]] = None, - backoff_factor: float = 2.0 + backoff_factor: float = 2.0, + debug: bool = False ): """ Initialize the AsyncHttpClient instance. @@ -58,6 +60,9 @@ def __init__( self.client = httpx.AsyncClient(timeout=self.timeout, verify=self.verify_ssl, headers=self.default_headers, auth=self.auth) + + if not debug: + logging.getLogger("httpx").setLevel(logging.WARNING) async def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False) -> str: # build URL Specification From f8f4d968cf620cd73b80e41ede947bb16776ce0c Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Sat, 28 Oct 2023 13:37:19 +0200 Subject: [PATCH 25/56] add retry warning message --- src/keboola/http_client/async_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index bdd4942..6ffe0f4 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -60,7 +60,7 @@ def __init__( self.client = httpx.AsyncClient(timeout=self.timeout, verify=self.verify_ssl, headers=self.default_headers, auth=self.auth) - + if not debug: logging.getLogger("httpx").setLevel(logging.WARNING) @@ -144,7 +144,7 @@ async def _request( return response - except httpx.HTTPError: + except httpx.HTTPError as e: if response: if response.status_code not in self.retry_status_codes: raise @@ -154,6 +154,8 @@ async def _request( backoff = self.backoff_factor ** retry_attempt await asyncio.sleep(backoff) + logging.warning(f"Retry attempt {retry_attempt + 1} for {method} request to {url}: {e}") + async def get(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, Any]: response = await self.get_raw(endpoint, **kwargs) return response.json() From 1ccea9165f7cc9997410b5accaa7773c62b6bd26 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Mon, 30 Oct 2023 10:37:02 +0100 Subject: [PATCH 26/56] reduce debug messages --- src/keboola/http_client/async_client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 6ffe0f4..b853953 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -61,8 +61,7 @@ def __init__( self.client = httpx.AsyncClient(timeout=self.timeout, verify=self.verify_ssl, headers=self.default_headers, auth=self.auth) - if not debug: - logging.getLogger("httpx").setLevel(logging.WARNING) + logging.getLogger("httpx").setLevel(logging.WARNING) async def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False) -> str: # build URL Specification @@ -142,6 +141,10 @@ async def _request( response.raise_for_status() + log_message = (f"HTTP Request: {method} {url} \"{response.http_version} {response.status_code} " + f"{response.reason_phrase}\"") + logging.debug(log_message) + return response except httpx.HTTPError as e: From 2acc6bcf773147ff78f93bc7a4eb2ce67e1e0a2d Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Mon, 30 Oct 2023 10:43:50 +0100 Subject: [PATCH 27/56] remove debug param --- src/keboola/http_client/async_client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index b853953..bc5713a 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -23,8 +23,7 @@ def __init__( auth: Optional[tuple] = None, auth_header: Optional[Dict[str, str]] = None, default_headers: Optional[Dict[str, str]] = None, - backoff_factor: float = 2.0, - debug: bool = False + backoff_factor: float = 2.0 ): """ Initialize the AsyncHttpClient instance. From e269ddf5ff120a3928f7b5ed73c02ef6555bb698 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Mon, 30 Oct 2023 10:53:11 +0100 Subject: [PATCH 28/56] revert logging setup --- src/keboola/http_client/async_client.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index bc5713a..6ffe0f4 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -23,7 +23,8 @@ def __init__( auth: Optional[tuple] = None, auth_header: Optional[Dict[str, str]] = None, default_headers: Optional[Dict[str, str]] = None, - backoff_factor: float = 2.0 + backoff_factor: float = 2.0, + debug: bool = False ): """ Initialize the AsyncHttpClient instance. @@ -60,7 +61,8 @@ def __init__( self.client = httpx.AsyncClient(timeout=self.timeout, verify=self.verify_ssl, headers=self.default_headers, auth=self.auth) - logging.getLogger("httpx").setLevel(logging.WARNING) + if not debug: + logging.getLogger("httpx").setLevel(logging.WARNING) async def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False) -> str: # build URL Specification @@ -140,10 +142,6 @@ async def _request( response.raise_for_status() - log_message = (f"HTTP Request: {method} {url} \"{response.http_version} {response.status_code} " - f"{response.reason_phrase}\"") - logging.debug(log_message) - return response except httpx.HTTPError as e: From 90edc2a2feb8ee22f62f57d8120b83c9fd1eaaf3 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Mon, 30 Oct 2023 10:56:21 +0100 Subject: [PATCH 29/56] silence httpcore debug messages --- src/keboola/http_client/async_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 6ffe0f4..47cabab 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -63,6 +63,7 @@ def __init__( if not debug: logging.getLogger("httpx").setLevel(logging.WARNING) + logging.getLogger("httpcore").setLevel(logging.WARNING) async def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False) -> str: # build URL Specification From 782f4d5ef8364eb542fd5e00323ffbd11a7f7847 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Mon, 29 Jul 2024 11:00:41 +0200 Subject: [PATCH 30/56] reenable debug --- src/keboola/http_client/async_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 47cabab..32fd1f1 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -61,10 +61,12 @@ def __init__( self.client = httpx.AsyncClient(timeout=self.timeout, verify=self.verify_ssl, headers=self.default_headers, auth=self.auth) + """ if not debug: logging.getLogger("httpx").setLevel(logging.WARNING) logging.getLogger("httpcore").setLevel(logging.WARNING) - + """ + async def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False) -> str: # build URL Specification url_path = str(endpoint_path).strip() if endpoint_path is not None else '' From c66364657111ae8e6ad020aa0e2f60fbd24e5623 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Mon, 29 Jul 2024 15:26:18 +0200 Subject: [PATCH 31/56] Update async_client.py --- src/keboola/http_client/async_client.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 32fd1f1..62faa8d 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -61,12 +61,10 @@ def __init__( self.client = httpx.AsyncClient(timeout=self.timeout, verify=self.verify_ssl, headers=self.default_headers, auth=self.auth) - """ if not debug: logging.getLogger("httpx").setLevel(logging.WARNING) logging.getLogger("httpcore").setLevel(logging.WARNING) - """ - + async def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False) -> str: # build URL Specification url_path = str(endpoint_path).strip() if endpoint_path is not None else '' @@ -157,7 +155,8 @@ async def _request( backoff = self.backoff_factor ** retry_attempt await asyncio.sleep(backoff) - logging.warning(f"Retry attempt {retry_attempt + 1} for {method} request to {url}: {e}") + message = response.text if response.text else str(e) + logging.error(f"Retry attempt {retry_attempt + 1} for {method} request to {url}: {message}") async def get(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, Any]: response = await self.get_raw(endpoint, **kwargs) From 91f52335aa40b0d49410da14d090745f5d2a6811 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Mon, 29 Jul 2024 15:35:37 +0200 Subject: [PATCH 32/56] Update async_client.py --- src/keboola/http_client/async_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 62faa8d..9f67847 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -155,7 +155,7 @@ async def _request( backoff = self.backoff_factor ** retry_attempt await asyncio.sleep(backoff) - message = response.text if response.text else str(e) + message = response.text if response and response.text else str(e) logging.error(f"Retry attempt {retry_attempt + 1} for {method} request to {url}: {message}") async def get(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, Any]: From 4f1b06159f6eec72daee1b0f4a7ffb4393d6d817 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Mon, 29 Jul 2024 16:03:13 +0200 Subject: [PATCH 33/56] bump httpx version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d8e5962..af4b5ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ requests -httpx==0.24.0 +httpx==0.27.0 aiolimiter==1.1.0 From cfe3c9b60d718f56887ab57467cef30afaa7d610 Mon Sep 17 00:00:00 2001 From: SgtMarmite Date: Thu, 12 Sep 2024 13:15:43 +0200 Subject: [PATCH 34/56] make max_requests per second float --- src/keboola/http_client/async_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 9f67847..7a855e5 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -18,7 +18,7 @@ def __init__( timeout: Optional[float] = None, verify_ssl: bool = True, retry_status_codes: Optional[List[int]] = None, - max_requests_per_second: Optional[int] = None, + max_requests_per_second: Optional[float] = None, default_params: Optional[Dict[str, str]] = None, auth: Optional[tuple] = None, auth_header: Optional[Dict[str, str]] = None, @@ -35,7 +35,7 @@ def __init__( timeout (Optional[float], optional): The request timeout in seconds. Defaults to None. verify_ssl (bool, optional): Enable or disable SSL verification. Defaults to True. retry_status_codes (Optional[List[int]], optional): List of status codes to retry on. Defaults to None. - max_requests_per_second (Optional[int], optional): Maximum number of requests per second. Defaults to None. + max_requests_per_second (Optional[float], optional): Maximum number of requests per second. Defaults to None. default_params (Optional[Dict[str, str]], optional): Default query parameters for each request. auth (Optional[tuple], optional): Authentication credentials for each request. Defaults to None. auth_header (Optional[Dict[str, str]], optional): Authentication header for each request. Defaults to None. @@ -52,7 +52,7 @@ def __init__( self.limiter = None if max_requests_per_second: - one_reqeust_per_second_amount = float(1/int(max_requests_per_second)) + one_reqeust_per_second_amount = float(1/max_requests_per_second) self.limiter = AsyncLimiter(1, one_reqeust_per_second_amount) self.default_headers = default_headers or {} From 6ec6c37a425be947c9ff01cbb908004023b74a74 Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Thu, 7 Nov 2024 13:03:12 +0100 Subject: [PATCH 35/56] added detailed exception message --- src/keboola/http_client/async_client.py | 5 ++++- tests/test_async.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 7a855e5..41bea34 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -146,6 +146,10 @@ async def _request( return response except httpx.HTTPError as e: + + message = response.text if response and response.text else str(e) + e.args = (f"Error '{e.response.status_code} {message}' for url '{e.request.url}'",) + if response: if response.status_code not in self.retry_status_codes: raise @@ -155,7 +159,6 @@ async def _request( backoff = self.backoff_factor ** retry_attempt await asyncio.sleep(backoff) - message = response.text if response and response.text else str(e) logging.error(f"Retry attempt {retry_attempt + 1} for {method} request to {url}: {message}") async def get(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, Any]: diff --git a/tests/test_async.py b/tests/test_async.py index 5c9618a..a77d921 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -294,6 +294,17 @@ async def test_update_existing_auth_header(self): await cl.update_auth_header(new_header, overwrite=False) self.assertDictEqual(cl._auth_header, {**existing_header, **new_header}) + async def test_detailed_exception(self): + mock_response = httpx.Response(404, text="Not Found Because of x") + mock_response._request = httpx.Request("GET", "https://api.example.com/endpoint") + + client = AsyncHttpClient(self.base_url) + + with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: + with self.assertRaises(httpx.HTTPStatusError) as e: + await client.get("/endpoint") + + assert "Error '404 Not Found Because of x' for url 'https://api.example.com/endpoint'" in str(e.exception) if __name__ == "__main__": unittest.main() From 6c59f8ab9745c5411cf9b218c72c164d6d4edb81 Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Thu, 7 Nov 2024 15:48:46 +0100 Subject: [PATCH 36/56] added detailed exception message --- src/keboola/http_client/async_client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 41bea34..ff93494 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -147,8 +147,9 @@ async def _request( except httpx.HTTPError as e: - message = response.text if response and response.text else str(e) - e.args = (f"Error '{e.response.status_code} {message}' for url '{e.request.url}'",) + if not isinstance(e, httpx.ReadTimeout): + message = response.text if response and response.text else str(e) + e.args = (f"Error '{e.response.status_code} {message}' for url '{e.request.url}'",) if response: if response.status_code not in self.retry_status_codes: From 9822f40bbeaa0a84d205a2671b21a44749b136e5 Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Thu, 7 Nov 2024 16:01:28 +0100 Subject: [PATCH 37/56] switch to python 3.12 --- .github/workflows/deploy.yml | 2 +- .github/workflows/deploy_to_test.yml | 2 +- .github/workflows/push_dev.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ae74d2e..e0faf35 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.12' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/deploy_to_test.yml b/.github/workflows/deploy_to_test.yml index 0a61c7f..0472b35 100644 --- a/.github/workflows/deploy_to_test.yml +++ b/.github/workflows/deploy_to_test.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.12' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/push_dev.yml b/.github/workflows/push_dev.yml index 6ff22ce..71cf2e7 100644 --- a/.github/workflows/push_dev.yml +++ b/.github/workflows/push_dev.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ 3.7 ] + python-version: [ 3.12 ] steps: - uses: actions/checkout@v2 From ff214ab84c2d26a8e924e2ec5ceba978a4ea7535 Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Thu, 7 Nov 2024 16:07:18 +0100 Subject: [PATCH 38/56] flake8 --- src/keboola/http_client/async_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index ff93494..84b2f21 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -11,6 +11,7 @@ class AsyncHttpClient: An asynchronous HTTP client that simplifies making requests to a specific API. """ ALLOWED_METHODS = ['GET', 'POST', 'PATCH', 'UPDATE', 'PUT', 'DELETE'] + def __init__( self, base_url: str, @@ -35,7 +36,7 @@ def __init__( timeout (Optional[float], optional): The request timeout in seconds. Defaults to None. verify_ssl (bool, optional): Enable or disable SSL verification. Defaults to True. retry_status_codes (Optional[List[int]], optional): List of status codes to retry on. Defaults to None. - max_requests_per_second (Optional[float], optional): Maximum number of requests per second. Defaults to None. + max_requests_per_second (Optional[float], optional): Maximum no. of requests per second. Defaults to None. default_params (Optional[Dict[str, str]], optional): Default query parameters for each request. auth (Optional[tuple], optional): Authentication credentials for each request. Defaults to None. auth_header (Optional[Dict[str, str]], optional): Authentication header for each request. Defaults to None. @@ -52,7 +53,7 @@ def __init__( self.limiter = None if max_requests_per_second: - one_reqeust_per_second_amount = float(1/max_requests_per_second) + one_reqeust_per_second_amount = float(1 / max_requests_per_second) self.limiter = AsyncLimiter(1, one_reqeust_per_second_amount) self.default_headers = default_headers or {} @@ -112,7 +113,6 @@ async def _request( **kwargs ) -> httpx.Response: - is_absolute_path = kwargs.pop('is_absolute_path', False) url = await self._build_url(endpoint, is_absolute_path) From 219f0f89455c55fb82dd1637a07c2cd5c76b7149 Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:34:51 +0400 Subject: [PATCH 39/56] fix message --- src/keboola/http_client/async_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 84b2f21..15df1fe 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -146,9 +146,9 @@ async def _request( return response except httpx.HTTPError as e: + message = response.text if response and response.text else str(e) if not isinstance(e, httpx.ReadTimeout): - message = response.text if response and response.text else str(e) e.args = (f"Error '{e.response.status_code} {message}' for url '{e.request.url}'",) if response: From b9876637c5aa5021c4dae7dc440634469054ca39 Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:59:54 +0400 Subject: [PATCH 40/56] resolve conflicts --- src/keboola/http_client/__init__.py | 2 +- src/keboola/http_client/{client.py => http.py} | 0 tests/test_http.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/keboola/http_client/{client.py => http.py} (100%) diff --git a/src/keboola/http_client/__init__.py b/src/keboola/http_client/__init__.py index 05f88ab..0ec7a35 100644 --- a/src/keboola/http_client/__init__.py +++ b/src/keboola/http_client/__init__.py @@ -1,2 +1,2 @@ -from .client import HttpClient # noqa +from .http import HttpClient # noqa from .async_client import AsyncHttpClient # noqa diff --git a/src/keboola/http_client/client.py b/src/keboola/http_client/http.py similarity index 100% rename from src/keboola/http_client/client.py rename to src/keboola/http_client/http.py diff --git a/tests/test_http.py b/tests/test_http.py index 82752b9..99dc66d 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -2,7 +2,7 @@ import urllib.parse as urlparse from unittest.mock import patch -import keboola.http_client.client as client +import keboola.http_client.http as client class TestClientBase(unittest.TestCase): From 84531f32f423581aa6c0285bdcb21df0089abecd Mon Sep 17 00:00:00 2001 From: Soustruh Date: Tue, 14 Jan 2025 09:47:48 +0100 Subject: [PATCH 41/56] HTTPError has no attribute response --- src/keboola/http_client/async_client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 15df1fe..9f318af 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -46,7 +46,7 @@ def __init__( self.retries = retries self.timeout = httpx.Timeout(timeout) if timeout else None self.verify_ssl = verify_ssl - self.retry_status_codes = retry_status_codes or [429, 500, 502, 504] + self.retry_status_codes = set(retry_status_codes) if retry_status_codes else {429, 500, 502, 504} self.default_params = default_params or {} self.auth = auth self._auth_header = auth_header or {} @@ -146,14 +146,14 @@ async def _request( return response except httpx.HTTPError as e: + st_code = response.status_code if response else 0 message = response.text if response and response.text else str(e) if not isinstance(e, httpx.ReadTimeout): - e.args = (f"Error '{e.response.status_code} {message}' for url '{e.request.url}'",) + e.args = (f"Error '{st_code} {message}' for url '{e.request.url}'",) - if response: - if response.status_code not in self.retry_status_codes: - raise + if st_code not in self.retry_status_codes: + raise if retry_attempt == self.retries: raise From bb2987b2965f3d397feaf67a1806a226e552dc5b Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Mon, 10 Mar 2025 12:28:44 +0100 Subject: [PATCH 42/56] retry on httpx.HTTPError, httpx.ReadError, httpx.ConnectError, httpx.ReadTimeout --- src/keboola/http_client/async_client.py | 32 +++++++++++++++++-------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 9f318af..ec2b970 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -145,23 +145,35 @@ async def _request( return response - except httpx.HTTPError as e: - st_code = response.status_code if response else 0 - message = response.text if response and response.text else str(e) + except (httpx.HTTPError, httpx.ReadError, httpx.ConnectError, httpx.ReadTimeout) as e: + if isinstance(e, httpx.HTTPStatusError) and response: + st_code = response.status_code + message = response.text if response.text else str(e) - if not isinstance(e, httpx.ReadTimeout): - e.args = (f"Error '{st_code} {message}' for url '{e.request.url}'",) + if st_code not in self.retry_status_codes: + raise + else: + message = str(e) - if st_code not in self.retry_status_codes: - raise + if hasattr(e, 'request') and e.request: + error_msg = f"Error '{message}' for url '{e.request.url}'" + else: + error_msg = f"Error '{message}' for url '{url}'" if retry_attempt == self.retries: - raise + if isinstance(e, httpx.HTTPStatusError): + raise + else: + raise type(e)(error_msg) from e + backoff = self.backoff_factor ** retry_attempt + logging.error( + f"Retry attempt {retry_attempt + 1} for {method} request to {url}: " + f"Exception={type(e).__name__}, Message='{message}', " + f"Params={all_params}" + ) await asyncio.sleep(backoff) - logging.error(f"Retry attempt {retry_attempt + 1} for {method} request to {url}: {message}") - async def get(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, Any]: response = await self.get_raw(endpoint, **kwargs) return response.json() From e561c69ff46ee2b899572cc7789fc4033f6b8301 Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Mon, 10 Mar 2025 15:42:36 +0100 Subject: [PATCH 43/56] test --- tests/test_async.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_async.py b/tests/test_async.py index a77d921..a72c9d2 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -304,7 +304,7 @@ async def test_detailed_exception(self): with self.assertRaises(httpx.HTTPStatusError) as e: await client.get("/endpoint") - assert "Error '404 Not Found Because of x' for url 'https://api.example.com/endpoint'" in str(e.exception) + assert "Client error '404 Not Found' for url 'https://api.example.com/endpoint'" in str(e.exception) if __name__ == "__main__": unittest.main() From e750fe205e320eda4a7b025091eb8b0606defb96 Mon Sep 17 00:00:00 2001 From: soustruh Date: Wed, 1 Oct 2025 18:01:16 +0200 Subject: [PATCH 44/56] simplified according to our actual needs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit aligned with the latest changes in the cookiecutter template 🍪 --- .gitignore | 151 ++++------------------------------------------------- 1 file changed, 9 insertions(+), 142 deletions(-) diff --git a/.gitignore b/.gitignore index 28d81a5..df68cdb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,145 +1,12 @@ -# Byte-compiled / optimized / DLL files __pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - - -src/test.py - - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# VSCode -.vscode/ - -# IntelliJ -.idea/ - -# MacOS files .DS_Store +.idea +.ruff_cache +.venv +.vscode +*.py[cod] +*.egg-info +venv -# Local test script -/docs/examples/test.py +# kbc datafolder +data From 801455c6f4834f06ccd21f5adf52c58999032d71 Mon Sep 17 00:00:00 2001 From: soustruh Date: Wed, 1 Oct 2025 18:02:07 +0200 Subject: [PATCH 45/56] minor wording changes --- README.md | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0e8aa67..0eaba53 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,20 @@ +- [Python HTTP Client](#python-http-client) + - [Introduction](#introduction) + - [Links](#links) + - [Quick Start](#quick-start) + - [Installation](#installation) + - [Structure and Functionality](#structure-and-functionality) + - [`HttpClient`](#httpclient) + - [Initialization](#initialization) + - [Default arguments](#default-arguments) + - [Basic authentication](#basic-authentication) + - [Simple POST request](#simple-post-request) + - [Working with URL paths](#working-with-url-paths) + - [Raw request Example](#raw-request-example) + - [Building HTTP client based on HTTPClient Example](#building-http-client-based-on-httpclient-example) + - [Async Usage](#async-usage) + - [Building HTTP client based on AsyncHttpClient Example](#building-http-client-based-on-asynchttpclient-example) + # Python HTTP Client ## Introduction @@ -207,15 +224,14 @@ print(cl.get_files()) ## Async Usage -The package also provides an asynchronous version of the HTTP client called AsyncHttpClient. +The package also provides an asynchronous version of the HTTP client called AsyncHttpClient. It allows you to make asynchronous requests using async/await syntax. To use the AsyncHttpClient, import it from keboola.http_client_async: ```python from keboola.http_client import AsyncHttpClient ``` -The AsyncHttpClient class provides similar functionality as the HttpClient class, but with asynchronous methods such as get, post, put, patch, and delete that return awaitable coroutines. -You can use these methods within async functions to perform non-blocking HTTP requests. +The AsyncHttpClient class provides functionality similar to the HttpClient class, but with asynchronous methods such as get, post, put, patch, and delete that return awaitable coroutines. You can use these methods within async functions to perform non-blocking HTTP requests. ```python import asyncio @@ -235,11 +251,11 @@ async def main(): asyncio.run(main()) ``` -The AsyncHttpClient provides similar initialization and request methods as the HttpClient. +The AsyncHttpClient provides initialization and request methods similar to the HttpClient. The request methods return awaitable coroutines that can be awaited in an asynchronous context. #### Building HTTP client based on AsyncHttpClient Example -This example demonstrates the default use of the HTTPClient as a base for REST API clients. +This example demonstrates the default use of the AsyncHttpClient as a base for REST API clients. ```python import asyncio From fa168772365d4980e1413e74dba384608458915f Mon Sep 17 00:00:00 2001 From: soustruh Date: Wed, 1 Oct 2025 18:12:43 +0200 Subject: [PATCH 46/56] =?UTF-8?q?ruff=20formatting=20in=20code=20excerpts?= =?UTF-8?q?=20=F0=9F=90=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 125 +++++++++++++++++++++++++++++------------------------- 1 file changed, 68 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 0eaba53..ec88d1b 100644 --- a/README.md +++ b/README.md @@ -69,10 +69,10 @@ All abovementioned methods support all parameters supported by `requests.request The core class is `keboola.http_client.HttpClient`, which can be initialized by specifying the `base_url` parameter: -```python +```py from keboola.http_client import HttpClient -BASE_URL = 'https://connection.keboola.com/v2/storage/' +BASE_URL = "https://connection.keboola.com/v2/storage/" cl = HttpClient(BASE_URL) ``` @@ -80,34 +80,32 @@ cl = HttpClient(BASE_URL) For `HttpClient`, it is possible to define default arguments, which will be sent with every request. It's possible to define `default_http_header`, `auth_header` and `default_params` - a default header, a default authentication header and default parameters, respectively. -```python +```py from keboola.http_client import HttpClient -BASE_URL = 'https://connection.keboola.com/v2/storage/' -AUTH_HEADER = { - 'x-storageapi-token': '1234-STORAGETOKENSTRING' -} -DEFAULT_PARAMS = { - 'include': 'columns' -} -DEFAULT_HEADER = { - 'Content-Type': 'application/json' -} - -cl = HttpClient(BASE_URL, default_http_header=DEFAULT_HEADER, - auth_header=AUTH_HEADER, default_params=DEFAULT_PARAMS) +BASE_URL = "https://connection.keboola.com/v2/storage/" +AUTH_HEADER = {"x-storageapi-token": "1234-STORAGETOKENSTRING"} +DEFAULT_PARAMS = {"include": "columns"} +DEFAULT_HEADER = {"Content-Type": "application/json"} + +cl = HttpClient( + BASE_URL, + default_http_header=DEFAULT_HEADER, + auth_header=AUTH_HEADER, + default_params=DEFAULT_PARAMS, +) ``` #### Basic authentication By specifying the `auth` argument, the `HttpClient` will utilize the basic authentication. -```python +```py from keboola.http_client import HttpClient -BASE_URL = 'https://connection.keboola.com/v2/storage/' -USERNAME = 'TestUser' -PASSWORD = '@bcd1234' +BASE_URL = "https://connection.keboola.com/v2/storage/" +USERNAME = "TestUser" +PASSWORD = "@bcd1234" cl = HttpClient(BASE_URL, auth=(USERNAME, PASSWORD)) ``` @@ -116,14 +114,14 @@ cl = HttpClient(BASE_URL, auth=(USERNAME, PASSWORD)) Making a simple POST request using `post_raw()` method. -```python +```py from keboola.http_client import HttpClient -BASE_URL = 'https://www.example.com/change' +BASE_URL = "https://www.example.com/change" cl = HttpClient(BASE_URL) -data = {'attr_1': 'value_1', 'attr_2': 'value_2'} -header = {'content-type': 'application/json'} +data = {"attr_1": "value_1", "attr_2": "value_2"} +header = {"content-type": "application/json"} response = cl.post_raw(data=data, headers=header) if response.ok is not True: @@ -134,14 +132,14 @@ else: Making a simple POST request using `post()` method. -```python +```py from keboola.http_client import HttpClient -BASE_URL = 'https://www.example.com/change' +BASE_URL = "https://www.example.com/change" cl = HttpClient(BASE_URL) -data = {'attr_1': 'value_1', 'attr_2': 'value_2'} -header = {'content-type': 'application/json'} +data = {"attr_1": "value_1", "attr_2": "value_2"} +header = {"content-type": "application/json"} response = cl.post(data=data, headers=header) ``` @@ -151,45 +149,45 @@ Each of the methods takes an optional positional argument `endpoint_path`. If sp The below code will send a POST request to the URL `https://example.com/api/v1/events`: -```python +```py from keboola.http_client import HttpClient -BASE_URL = 'https://example.com/api/v1' +BASE_URL = "https://example.com/api/v1" cl = HttpClient(BASE_URL) -header = {'token': 'token_value'} -cl.post_raw('events', headers=header) +header = {"token": "token_value"} +cl.post_raw("events", headers=header) ``` It is also possible to override this behavior by using parameter `is_absolute_path=True`. If specified, the value of `endpoint_path` will not be appended to the `base_url` parameter, but will rather be used as an absolute URL to which the HTTP request will be made. In the below code, the `base_url` parameter is set to `https://example.com/api/v1`, but the base URL will be overriden by specifying `is_absolute_path=True` and the HTTP request will be made to the URL specified in the `post()` request - `https://anothersite.com/v2`. -```python +```py from keboola.http_client import HttpClient -BASE_URL = 'https://example.com/api/v1' +BASE_URL = "https://example.com/api/v1" cl = HttpClient(BASE_URL) -header = {'token': 'token_value'} -cl.post_raw('https://anothersite.com/v2', headers=header, is_absolute_path=True) +header = {"token": "token_value"} +cl.post_raw("https://anothersite.com/v2", headers=header, is_absolute_path=True) ``` #### Raw request Example A simple request made with default authentication header and parameters. -```python +```py import os from keboola.http_client import HttpClient -BASE_URL = 'https://connection.keboola.com/v2/' -TOKEN = os.environ['TOKEN'] +BASE_URL = "https://connection.keboola.com/v2/" +TOKEN = os.environ["TOKEN"] -cl = HttpClient(BASE_URL, auth_header={'x-storageapi-token': TOKEN}) +cl = HttpClient(BASE_URL, auth_header={"x-storageapi-token": TOKEN}) -request_params = {'exclude': 'components'} -response = cl.get_raw('storage', params=request_params) +request_params = {"exclude": "components"} +response = cl.get_raw("storage", params=request_params) if response.ok is True: print(response.json()) @@ -199,23 +197,28 @@ if response.ok is True: This example demonstrates the default use of the HTTPClient as a base for REST API clients. -```python +```py from keboola.http_client import HttpClient -BASE_URL = 'https://connection.eu-central-1.keboola.com/v2/storage' +BASE_URL = "https://connection.eu-central-1.keboola.com/v2/storage" MAX_RETRIES = 10 class KBCStorageClient(HttpClient): - def __init__(self, storage_token): - HttpClient.__init__(self, base_url=BASE_URL, max_retries=MAX_RETRIES, backoff_factor=0.3, - status_forcelist=(429, 500, 502, 504), - auth_header={"X-StorageApi-Token": storage_token}) + HttpClient.__init__( + self, + base_url=BASE_URL, + max_retries=MAX_RETRIES, + backoff_factor=0.3, + status_forcelist=(429, 500, 502, 504), + auth_header={"X-StorageApi-Token": storage_token}, + ) def get_files(self, show_expired=False): params = {"showExpired": show_expired} - return self.get('files', params=params) + return self.get("files", params=params) + cl = KBCStorageClient("my_token") @@ -227,16 +230,18 @@ print(cl.get_files()) The package also provides an asynchronous version of the HTTP client called AsyncHttpClient. It allows you to make asynchronous requests using async/await syntax. To use the AsyncHttpClient, import it from keboola.http_client_async: -```python +```py from keboola.http_client import AsyncHttpClient ``` The AsyncHttpClient class provides functionality similar to the HttpClient class, but with asynchronous methods such as get, post, put, patch, and delete that return awaitable coroutines. You can use these methods within async functions to perform non-blocking HTTP requests. -```python +```py import asyncio + from keboola.http_client import AsyncHttpClient + async def main(): base_url = "https://api.example.com/" async with AsyncHttpClient(base_url) as client: @@ -247,6 +252,8 @@ async def main(): # Process the response data else: # Handle the error + pass + asyncio.run(main()) ``` @@ -257,15 +264,16 @@ The request methods return awaitable coroutines that can be awaited in an asynch #### Building HTTP client based on AsyncHttpClient Example This example demonstrates the default use of the AsyncHttpClient as a base for REST API clients. -```python +```py import asyncio + from keboola.http_client import AsyncHttpClient -BASE_URL = 'https://connection.keboola.com/v2/storage' +BASE_URL = "https://connection.keboola.com/v2/storage" MAX_RETRIES = 3 -class KBCStorageClient(AsyncHttpClient): +class KBCStorageClient(AsyncHttpClient): def __init__(self, storage_token): AsyncHttpClient.__init__( self, @@ -273,20 +281,23 @@ class KBCStorageClient(AsyncHttpClient): retries=MAX_RETRIES, backoff_factor=0.3, retry_status_codes=[429, 500, 502, 504], - auth_header={"X-StorageApi-Token": storage_token} + auth_header={"X-StorageApi-Token": storage_token}, ) async def get_files(self, show_expired=False): params = {"showExpired": show_expired} - response = await self.get('tables', params=params, timeout=5) + response = await self.get("tables", params=params, timeout=5) return response + async def main(): cl = KBCStorageClient("my_token") files = await cl.get_files(show_expired=False) print(files) + asyncio.run(main()) ``` + **Note:** Since there are no parallel requests being made, you won't notice any speedup for this use case. -For an example where you can see the speedup thanks to async requests, you can view the pokeapi.py in docs/examples. +For an example of a noticeable speedup thanks to async requests, see the pokeapi.py in `docs/examples`. From 6bd3c2727cb8f3526be65b1a35588f20df401634 Mon Sep 17 00:00:00 2001 From: soustruh Date: Wed, 1 Oct 2025 18:14:49 +0200 Subject: [PATCH 47/56] =?UTF-8?q?filename=20typo=20=E2=9A=A1=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/examples/{poekapi_async.py => pokeapi_async.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/examples/{poekapi_async.py => pokeapi_async.py} (100%) diff --git a/docs/examples/poekapi_async.py b/docs/examples/pokeapi_async.py similarity index 100% rename from docs/examples/poekapi_async.py rename to docs/examples/pokeapi_async.py From 5babc62a335e065dbd90fb159916fde7198d3396 Mon Sep 17 00:00:00 2001 From: soustruh Date: Wed, 1 Oct 2025 18:32:24 +0200 Subject: [PATCH 48/56] =?UTF-8?q?removed=20deprecated=20type=20annotations?= =?UTF-8?q?=20(from=20typing=20import=20=E2=80=A6)=20+=20ruff=20?= =?UTF-8?q?=F0=9F=90=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/examples/pokeapi_async.py | 20 +- docs/examples/pokeapi_process_multiple.py | 19 +- src/keboola/http_client/async_client.py | 116 +++--- src/keboola/http_client/http.py | 425 +++++++++++++++++----- 4 files changed, 412 insertions(+), 168 deletions(-) diff --git a/docs/examples/pokeapi_async.py b/docs/examples/pokeapi_async.py index 915cac0..6736293 100644 --- a/docs/examples/pokeapi_async.py +++ b/docs/examples/pokeapi_async.py @@ -1,9 +1,11 @@ -import time import asyncio -from keboola.http_client import AsyncHttpClient import csv -import httpx import os +import time + +import httpx + +from keboola.http_client import AsyncHttpClient async def fetch_pokemon(client, poke_id): @@ -30,11 +32,13 @@ async def save_to_csv(details): if not file_exists: writer.writeheader() - writer.writerow({ - "name": details["name"], - "height": details["height"], - "weight": details["weight"] - }) + writer.writerow( + { + "name": details["name"], + "height": details["height"], + "weight": details["weight"], + } + ) async def main_async(): diff --git a/docs/examples/pokeapi_process_multiple.py b/docs/examples/pokeapi_process_multiple.py index 1681877..815a718 100644 --- a/docs/examples/pokeapi_process_multiple.py +++ b/docs/examples/pokeapi_process_multiple.py @@ -1,15 +1,15 @@ import asyncio import csv import time -from typing import List from keboola.http_client import AsyncHttpClient def generate_jobs(nr_of_jobs): - return [{'method': 'GET', 'endpoint': str(endpoint)} for endpoint in range(1, nr_of_jobs+1)] + return [{"method": "GET", "endpoint": str(endpoint)} for endpoint in range(1, nr_of_jobs + 1)] -def save_to_csv(results: List[dict]): + +def save_to_csv(results: list[dict]): filename = "pokemon_details.csv" fieldnames = ["name", "height", "weight"] # Define the fields you want to store @@ -17,11 +17,14 @@ def save_to_csv(results: List[dict]): writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() for result in results: - writer.writerow({ - "name": result["name"], - "height": result["height"], - "weight": result["weight"] - }) + writer.writerow( + { + "name": result["name"], + "height": result["height"], + "weight": result["weight"], + } + ) + async def main_async(): base_url = "https://pokeapi.co/api/v2/pokemon/" diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index ec2b970..85b2cbd 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -1,31 +1,33 @@ -import httpx import asyncio -from typing import Optional, Dict, Any, List +import logging +from typing import Any from urllib.parse import urljoin + +import httpx from aiolimiter import AsyncLimiter -import logging class AsyncHttpClient: """ An asynchronous HTTP client that simplifies making requests to a specific API. """ - ALLOWED_METHODS = ['GET', 'POST', 'PATCH', 'UPDATE', 'PUT', 'DELETE'] + + ALLOWED_METHODS = ["GET", "POST", "PATCH", "UPDATE", "PUT", "DELETE"] def __init__( - self, - base_url: str, - retries: int = 3, - timeout: Optional[float] = None, - verify_ssl: bool = True, - retry_status_codes: Optional[List[int]] = None, - max_requests_per_second: Optional[float] = None, - default_params: Optional[Dict[str, str]] = None, - auth: Optional[tuple] = None, - auth_header: Optional[Dict[str, str]] = None, - default_headers: Optional[Dict[str, str]] = None, - backoff_factor: float = 2.0, - debug: bool = False + self, + base_url: str, + retries: int = 3, + timeout: float | None = None, + verify_ssl: bool = True, + retry_status_codes: list[int] | None = None, + max_requests_per_second: float | None = None, + default_params: dict[str, str] | None = None, + auth: tuple | None = None, + auth_header: dict[str, str] | None = None, + default_headers: dict[str, str] | None = None, + backoff_factor: float = 2.0, + debug: bool = False, ): """ Initialize the AsyncHttpClient instance. @@ -59,16 +61,17 @@ def __init__( self.default_headers = default_headers or {} self.backoff_factor = backoff_factor - self.client = httpx.AsyncClient(timeout=self.timeout, verify=self.verify_ssl, headers=self.default_headers, - auth=self.auth) + self.client = httpx.AsyncClient( + timeout=self.timeout, verify=self.verify_ssl, headers=self.default_headers, auth=self.auth + ) if not debug: logging.getLogger("httpx").setLevel(logging.WARNING) logging.getLogger("httpcore").setLevel(logging.WARNING) - async def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False) -> str: + async def _build_url(self, endpoint_path: str | None = None, is_absolute_path=False) -> str: # build URL Specification - url_path = str(endpoint_path).strip() if endpoint_path is not None else '' + url_path = str(endpoint_path).strip() if endpoint_path is not None else "" if not url_path: url = self.base_url @@ -79,7 +82,7 @@ async def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path return url - async def update_auth_header(self, updated_header: Dict, overwrite: bool = False): + async def update_auth_header(self, updated_header: dict, overwrite: bool = False): """ Updates the default auth header by providing new values. @@ -105,31 +108,30 @@ async def close(self): await self.client.aclose() async def _request( - self, - method: str, - endpoint: Optional[str] = None, - params: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, - **kwargs + self, + method: str, + endpoint: str | None = None, + params: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + **kwargs, ) -> httpx.Response: - - is_absolute_path = kwargs.pop('is_absolute_path', False) + is_absolute_path = kwargs.pop("is_absolute_path", False) url = await self._build_url(endpoint, is_absolute_path) all_params = {**self.default_params, **(params or {})} - ignore_auth = kwargs.pop('ignore_auth', False) + ignore_auth = kwargs.pop("ignore_auth", False) if ignore_auth: all_headers = {**self.default_headers, **(headers or {})} else: all_headers = {**self._auth_header, **self.default_headers, **(headers or {})} if self.auth: - kwargs.update({'auth': self.auth}) + kwargs.update({"auth": self.auth}) if all_params: - kwargs.update({'params': all_params}) + kwargs.update({"params": all_params}) if all_headers: - kwargs.update({'headers': all_headers}) + kwargs.update({"headers": all_headers}) response = None @@ -155,7 +157,7 @@ async def _request( else: message = str(e) - if hasattr(e, 'request') and e.request: + if hasattr(e, "request") and e.request: error_msg = f"Error '{message}' for url '{e.request.url}'" else: error_msg = f"Error '{message}' for url '{url}'" @@ -166,7 +168,7 @@ async def _request( else: raise type(e)(error_msg) from e - backoff = self.backoff_factor ** retry_attempt + backoff = self.backoff_factor**retry_attempt logging.error( f"Retry attempt {retry_attempt + 1} for {method} request to {url}: " f"Exception={type(e).__name__}, Message='{message}', " @@ -174,72 +176,72 @@ async def _request( ) await asyncio.sleep(backoff) - async def get(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, Any]: + async def get(self, endpoint: str | None = None, **kwargs) -> dict[str, Any]: response = await self.get_raw(endpoint, **kwargs) return response.json() - async def get_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + async def get_raw(self, endpoint: str | None = None, **kwargs) -> httpx.Response: return await self._request("GET", endpoint, **kwargs) - async def post(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, Any]: + async def post(self, endpoint: str | None = None, **kwargs) -> dict[str, Any]: response = await self.post_raw(endpoint, **kwargs) return response.json() - async def post_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + async def post_raw(self, endpoint: str | None = None, **kwargs) -> httpx.Response: return await self._request("POST", endpoint, **kwargs) - async def put(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, Any]: + async def put(self, endpoint: str | None = None, **kwargs) -> dict[str, Any]: response = await self.put_raw(endpoint, **kwargs) return response.json() - async def put_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + async def put_raw(self, endpoint: str | None = None, **kwargs) -> httpx.Response: return await self._request("PUT", endpoint, **kwargs) - async def patch(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, Any]: + async def patch(self, endpoint: str | None = None, **kwargs) -> dict[str, Any]: response = await self.patch_raw(endpoint, **kwargs) return response.json() - async def patch_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + async def patch_raw(self, endpoint: str | None = None, **kwargs) -> httpx.Response: return await self._request("PATCH", endpoint, **kwargs) - async def delete(self, endpoint: Optional[str] = None, **kwargs) -> Dict[str, Any]: + async def delete(self, endpoint: str | None = None, **kwargs) -> dict[str, Any]: response = await self.delete_raw(endpoint, **kwargs) return response.json() - async def delete_raw(self, endpoint: Optional[str] = None, **kwargs) -> httpx.Response: + async def delete_raw(self, endpoint: str | None = None, **kwargs) -> httpx.Response: return await self._request("DELETE", endpoint, **kwargs) - async def process_multiple(self, jobs: List[Dict[str, Any]]): + async def process_multiple(self, jobs: list[dict[str, Any]]): tasks = [] for job in jobs: - method = job['method'] - endpoint = job['endpoint'] - params = job.get('params') - headers = job.get('headers') - raw = job.get('raw', False) + method = job["method"] + endpoint = job["endpoint"] + params = job.get("params") + headers = job.get("headers") + raw = job.get("raw", False) - if method == 'GET': + if method == "GET": if raw: task = self.get_raw(endpoint, params=params, headers=headers) else: task = self.get(endpoint, params=params, headers=headers) - elif method == 'POST': + elif method == "POST": if raw: task = self.post_raw(endpoint, params=params, headers=headers) else: task = self.post(endpoint, params=params, headers=headers) - elif method == 'PUT': + elif method == "PUT": if raw: task = self.put_raw(endpoint, params=params, headers=headers) else: task = self.put(endpoint, params=params, headers=headers) - elif method == 'PATCH': + elif method == "PATCH": if raw: task = self.patch_raw(endpoint, params=params, headers=headers) else: task = self.patch(endpoint, params=params, headers=headers) - elif method == 'DELETE': + elif method == "DELETE": if raw: task = self.delete_raw(endpoint, params=params, headers=headers) else: diff --git a/src/keboola/http_client/http.py b/src/keboola/http_client/http.py index d465391..fc81036 100644 --- a/src/keboola/http_client/http.py +++ b/src/keboola/http_client/http.py @@ -2,16 +2,15 @@ import logging import urllib.parse as urlparse from http.cookiejar import CookieJar -from typing import Dict, Union, Tuple, Optional import requests from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry # noqa -Cookie = Union[Dict[str, str], CookieJar] +Cookie = dict[str, str] | CookieJar -METHOD_RETRY_WHITELIST = ('GET', 'POST', 'PATCH', 'UPDATE', 'PUT', 'DELETE') -ALLOWED_METHODS = ['GET', 'POST', 'PATCH', 'UPDATE', 'PUT', 'DELETE'] +METHOD_RETRY_WHITELIST = ("GET", "POST", "PATCH", "UPDATE", "PUT", "DELETE") +ALLOWED_METHODS = ["GET", "POST", "PATCH", "UPDATE", "PUT", "DELETE"] class HttpClient: @@ -43,10 +42,18 @@ class HttpClient: """ - def __init__(self, base_url: str, max_retries: int = 10, backoff_factor: float = 0.3, - status_forcelist: Tuple[int, ...] = (500, 502, 504), default_http_header: Dict = None, - auth_header: Dict = None, auth: Tuple = None, default_params: Dict = None, - allowed_methods: Tuple = METHOD_RETRY_WHITELIST): + def __init__( + self, + base_url: str, + max_retries: int = 10, + backoff_factor: float = 0.3, + status_forcelist: tuple[int, ...] = (500, 502, 504), + default_http_header: dict | None = None, + auth_header: dict | None = None, + auth: tuple | None = None, + default_params: dict | None = None, + allowed_methods: tuple = METHOD_RETRY_WHITELIST, + ): """ Create an endpoint. @@ -70,7 +77,7 @@ def __init__(self, base_url: str, max_retries: int = 10, backoff_factor: float = if base_url is None: raise ValueError("Base URL is required.") # Add trailing slash because of nature of urllib.parse.urljoin() - self.base_url = base_url if base_url.endswith('/') else base_url + '/' + self.base_url = base_url if base_url.endswith("/") else base_url + "/" self.max_retries = max_retries self.backoff_factor = backoff_factor self.status_forcelist = status_forcelist @@ -88,16 +95,16 @@ def _requests_retry_session(self, session=None): connect=self.max_retries, backoff_factor=self.backoff_factor, status_forcelist=self.status_forcelist, - allowed_methods=self.allowed_methods + allowed_methods=self.allowed_methods, ) adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) + session.mount("http://", adapter) + session.mount("https://", adapter) return session - def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False): + def _build_url(self, endpoint_path: str | None = None, is_absolute_path=False): # build URL Specification - url_path = str(endpoint_path).strip() if endpoint_path is not None else '' + url_path = str(endpoint_path).strip() if endpoint_path is not None else "" if not url_path: url = self.base_url @@ -108,7 +115,7 @@ def _build_url(self, endpoint_path: Optional[str] = None, is_absolute_path=False return url - def _request_raw(self, method: str, endpoint_path: Optional[str] = None, **kwargs) -> requests.Response: + def _request_raw(self, method: str, endpoint_path: str | None = None, **kwargs) -> requests.Response: """ Construct a requests call with args and kwargs and process the results. @@ -130,11 +137,11 @@ def _request_raw(self, method: str, endpoint_path: Optional[str] = None, **kwarg s = requests.Session() # build URL Specification - is_absolute_path = kwargs.pop('is_absolute_path', False) + is_absolute_path = kwargs.pop("is_absolute_path", False) url = self._build_url(endpoint_path, is_absolute_path) # Update headers - headers = kwargs.pop('headers', {}) + headers = kwargs.pop("headers", {}) if headers is None: headers = {} @@ -142,7 +149,7 @@ def _request_raw(self, method: str, endpoint_path: Optional[str] = None, **kwarg headers.update(self._default_header) # Auth headers - if kwargs.pop('ignore_auth', False) is False: + if kwargs.pop("ignore_auth", False) is False: headers.update(self._auth_header) s.headers.update(headers) s.auth = self._auth @@ -150,24 +157,23 @@ def _request_raw(self, method: str, endpoint_path: Optional[str] = None, **kwarg s.headers.update(headers) # Update parameters - params = kwargs.pop('params', {}) + params = kwargs.pop("params", {}) if params is None: params = {} # Default parameters if self._default_params is not None: all_pars = {**params, **self._default_params} - kwargs.update({'params': all_pars}) + kwargs.update({"params": all_pars}) else: - kwargs.update({'params': params}) + kwargs.update({"params": params}) r = self._requests_retry_session(session=s).request(method, url, **kwargs) return r def response_error_handling(func): - """Function, that handles response handling of HTTP requests. - """ + """Function, that handles response handling of HTTP requests.""" @functools.wraps(func) def wrapper(*args, **kwargs): @@ -183,7 +189,7 @@ def wrapper(*args, **kwargs): return wrapper - def update_auth_header(self, updated_header: Dict, overwrite: bool = False): + def update_auth_header(self, updated_header: dict, overwrite: bool = False): """ Updates the default auth header by providing new values. @@ -198,9 +204,16 @@ def update_auth_header(self, updated_header: Dict, overwrite: bool = False): else: self._auth_header = updated_header - def get_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, - is_absolute_path: bool = False, cookies: Cookie = None, - ignore_auth: bool = False, **kwargs) -> requests.Response: + def get_raw( + self, + endpoint_path: str | None = None, + params: dict | None = None, + headers: dict | None = None, + is_absolute_path: bool = False, + cookies: Cookie | None = None, + ignore_auth: bool = False, + **kwargs, + ) -> requests.Response: """ Constructs a requests GET call with specified url and kwargs to process the result. @@ -230,14 +243,29 @@ def get_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, head A [`requests.Response`](https://requests.readthedocs.io/en/latest/api/#requests.Response) object. """ - method = 'GET' - return self._request_raw(method, endpoint_path, params=params, headers=headers, cookies=cookies, - is_absolute_path=is_absolute_path, ignore_auth=ignore_auth, **kwargs) + method = "GET" + return self._request_raw( + method, + endpoint_path, + params=params, + headers=headers, + cookies=cookies, + is_absolute_path=is_absolute_path, + ignore_auth=ignore_auth, + **kwargs, + ) @response_error_handling - def get(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, - is_absolute_path: bool = False, cookies: Cookie = None, - ignore_auth: bool = False, **kwargs) -> requests.Response: + def get( + self, + endpoint_path: str | None = None, + params: dict | None = None, + headers: dict | None = None, + is_absolute_path: bool = False, + cookies: Cookie | None = None, + ignore_auth: bool = False, + **kwargs, + ) -> requests.Response: """ Constructs a requests GET call with specified url and kwargs to process the result. @@ -271,12 +299,29 @@ def get(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: requests.HTTPError: If the API request fails. """ - return self.get_raw(endpoint_path, params=params, headers=headers, cookies=cookies, - is_absolute_path=is_absolute_path, ignore_auth=ignore_auth, **kwargs) + return self.get_raw( + endpoint_path, + params=params, + headers=headers, + cookies=cookies, + is_absolute_path=is_absolute_path, + ignore_auth=ignore_auth, + **kwargs, + ) - def post_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, - data: Dict = None, json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, - files: Dict = None, ignore_auth: bool = False, **kwargs) -> requests.Response: + def post_raw( + self, + endpoint_path: str | None = None, + params: dict | None = None, + headers: dict | None = None, + data: dict | None = None, + json: dict | None = None, + is_absolute_path: bool = False, + cookies: Cookie | None = None, + files: dict | None = None, + ignore_auth: bool = False, + **kwargs, + ) -> requests.Response: """ Constructs a requests POST call with specified url and kwargs to process the result. @@ -309,15 +354,35 @@ def post_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, hea A [`requests.Response`](https://requests.readthedocs.io/en/latest/api/#requests.Response) object. """ - method = 'POST' - return self._request_raw(method, endpoint_path, params=params, headers=headers, data=data, json=json, - cookies=cookies, is_absolute_path=is_absolute_path, files=files, - ignore_auth=ignore_auth, **kwargs) + method = "POST" + return self._request_raw( + method, + endpoint_path, + params=params, + headers=headers, + data=data, + json=json, + cookies=cookies, + is_absolute_path=is_absolute_path, + files=files, + ignore_auth=ignore_auth, + **kwargs, + ) @response_error_handling - def post(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, data: Dict = None, - json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, files: Dict = None, - ignore_auth: bool = False, **kwargs) -> requests.Response: + def post( + self, + endpoint_path: str | None = None, + params: dict | None = None, + headers: dict | None = None, + data: dict | None = None, + json: dict | None = None, + is_absolute_path: bool = False, + cookies: Cookie | None = None, + files: dict | None = None, + ignore_auth: bool = False, + **kwargs, + ) -> requests.Response: """ Constructs a requests POST call with specified url and kwargs to process the result. @@ -353,12 +418,32 @@ def post(self, endpoint_path: Optional[str] = None, params: Dict = None, headers requests.HTTPError: If the API request fails. """ - return self.post_raw(endpoint_path, params=params, headers=headers, data=data, json=json, cookies=cookies, - is_absolute_path=is_absolute_path, files=files, ignore_auth=ignore_auth, **kwargs) + return self.post_raw( + endpoint_path, + params=params, + headers=headers, + data=data, + json=json, + cookies=cookies, + is_absolute_path=is_absolute_path, + files=files, + ignore_auth=ignore_auth, + **kwargs, + ) - def patch_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, - data: Dict = None, json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, - files: Dict = None, ignore_auth: bool = False, **kwargs) -> requests.Response: + def patch_raw( + self, + endpoint_path: str | None = None, + params: dict | None = None, + headers: dict | None = None, + data: dict | None = None, + json: dict | None = None, + is_absolute_path: bool = False, + cookies: Cookie | None = None, + files: dict | None = None, + ignore_auth: bool = False, + **kwargs, + ) -> requests.Response: """ Constructs a requests PATCH call with specified url and kwargs to process the result. @@ -391,15 +476,35 @@ def patch_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, he A [`requests.Response`](https://requests.readthedocs.io/en/latest/api/#requests.Response) object. """ - method = 'PATCH' - return self._request_raw(method, endpoint_path, params=params, headers=headers, data=data, json=json, - cookies=cookies, is_absolute_path=is_absolute_path, files=files, - ignore_auth=ignore_auth, **kwargs) + method = "PATCH" + return self._request_raw( + method, + endpoint_path, + params=params, + headers=headers, + data=data, + json=json, + cookies=cookies, + is_absolute_path=is_absolute_path, + files=files, + ignore_auth=ignore_auth, + **kwargs, + ) @response_error_handling - def patch(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, data: Dict = None, - json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, files: Dict = None, - ignore_auth: bool = False, **kwargs) -> requests.Response: + def patch( + self, + endpoint_path: str | None = None, + params: dict | None = None, + headers: dict | None = None, + data: dict | None = None, + json: dict | None = None, + is_absolute_path: bool = False, + cookies: Cookie | None = None, + files: dict | None = None, + ignore_auth: bool = False, + **kwargs, + ) -> requests.Response: """ Constructs a requests PATCH call with specified url and kwargs to process the result. @@ -435,12 +540,32 @@ def patch(self, endpoint_path: Optional[str] = None, params: Dict = None, header requests.HTTPError: If the API request fails. """ - return self.patch_raw(endpoint_path, params=params, headers=headers, data=data, json=json, cookies=cookies, - is_absolute_path=is_absolute_path, files=files, ignore_auth=ignore_auth, **kwargs) + return self.patch_raw( + endpoint_path, + params=params, + headers=headers, + data=data, + json=json, + cookies=cookies, + is_absolute_path=is_absolute_path, + files=files, + ignore_auth=ignore_auth, + **kwargs, + ) - def update_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, - data: Dict = None, json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, - files: Dict = None, ignore_auth: bool = False, **kwargs) -> requests.Response: + def update_raw( + self, + endpoint_path: str | None = None, + params: dict | None = None, + headers: dict | None = None, + data: dict | None = None, + json: dict | None = None, + is_absolute_path: bool = False, + cookies: Cookie | None = None, + files: dict | None = None, + ignore_auth: bool = False, + **kwargs, + ) -> requests.Response: """ Constructs a requests UPDATE call with specified url and kwargs to process the result. @@ -473,15 +598,35 @@ def update_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, h A [`requests.Response`](https://requests.readthedocs.io/en/latest/api/#requests.Response) object. """ - method = 'UPDATE' - return self._request_raw(method, endpoint_path, params=params, headers=headers, data=data, json=json, - cookies=cookies, is_absolute_path=is_absolute_path, files=files, - ignore_auth=ignore_auth, **kwargs) + method = "UPDATE" + return self._request_raw( + method, + endpoint_path, + params=params, + headers=headers, + data=data, + json=json, + cookies=cookies, + is_absolute_path=is_absolute_path, + files=files, + ignore_auth=ignore_auth, + **kwargs, + ) @response_error_handling - def update(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, data: Dict = None, - json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, files: Dict = None, - ignore_auth: bool = False, **kwargs) -> requests.Response: + def update( + self, + endpoint_path: str | None = None, + params: dict | None = None, + headers: dict | None = None, + data: dict | None = None, + json: dict | None = None, + is_absolute_path: bool = False, + cookies: Cookie | None = None, + files: dict | None = None, + ignore_auth: bool = False, + **kwargs, + ) -> requests.Response: """ Constructs a requests UPDATE call with specified url and kwargs to process the result. @@ -517,12 +662,32 @@ def update(self, endpoint_path: Optional[str] = None, params: Dict = None, heade requests.HTTPError: If the API request fails. """ - return self.update_raw(endpoint_path, params=params, headers=headers, data=data, json=json, cookies=cookies, - is_absolute_path=is_absolute_path, files=files, ignore_auth=ignore_auth, **kwargs) + return self.update_raw( + endpoint_path, + params=params, + headers=headers, + data=data, + json=json, + cookies=cookies, + is_absolute_path=is_absolute_path, + files=files, + ignore_auth=ignore_auth, + **kwargs, + ) - def put_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, - data: Dict = None, json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, - files: Dict = None, ignore_auth: bool = False, **kwargs) -> requests.Response: + def put_raw( + self, + endpoint_path: str | None = None, + params: dict | None = None, + headers: dict | None = None, + data: dict | None = None, + json: dict | None = None, + is_absolute_path: bool = False, + cookies: Cookie | None = None, + files: dict | None = None, + ignore_auth: bool = False, + **kwargs, + ) -> requests.Response: """ Constructs a requests PUT call with specified url and kwargs to process the result. @@ -555,15 +720,35 @@ def put_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, head A [`requests.Response`](https://requests.readthedocs.io/en/latest/api/#requests.Response) object. """ - method = 'PUT' - return self._request_raw(method, endpoint_path, params=params, headers=headers, data=data, json=json, - cookies=cookies, is_absolute_path=is_absolute_path, files=files, - ignore_auth=ignore_auth, **kwargs) + method = "PUT" + return self._request_raw( + method, + endpoint_path, + params=params, + headers=headers, + data=data, + json=json, + cookies=cookies, + is_absolute_path=is_absolute_path, + files=files, + ignore_auth=ignore_auth, + **kwargs, + ) @response_error_handling - def put(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, data: Dict = None, - json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, files: Dict = None, - ignore_auth: bool = False, **kwargs) -> requests.Response: + def put( + self, + endpoint_path: str | None = None, + params: dict | None = None, + headers: dict | None = None, + data: dict | None = None, + json: dict | None = None, + is_absolute_path: bool = False, + cookies: Cookie | None = None, + files: dict | None = None, + ignore_auth: bool = False, + **kwargs, + ) -> requests.Response: """ Constructs a requests PUT call with specified url and kwargs to process the result. @@ -599,12 +784,32 @@ def put(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: requests.HTTPError: If the API request fails. """ - return self.put_raw(endpoint_path, params=params, headers=headers, data=data, json=json, cookies=cookies, - is_absolute_path=is_absolute_path, files=files, ignore_auth=ignore_auth, **kwargs) + return self.put_raw( + endpoint_path, + params=params, + headers=headers, + data=data, + json=json, + cookies=cookies, + is_absolute_path=is_absolute_path, + files=files, + ignore_auth=ignore_auth, + **kwargs, + ) - def delete_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, - data: Dict = None, json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, - files: Dict = None, ignore_auth: bool = False, **kwargs) -> requests.Response: + def delete_raw( + self, + endpoint_path: str | None = None, + params: dict | None = None, + headers: dict | None = None, + data: dict | None = None, + json: dict | None = None, + is_absolute_path: bool = False, + cookies: Cookie | None = None, + files: dict | None = None, + ignore_auth: bool = False, + **kwargs, + ) -> requests.Response: """ Constructs a requests DELETE call with specified url and kwargs to process the result. @@ -637,15 +842,35 @@ def delete_raw(self, endpoint_path: Optional[str] = None, params: Dict = None, h A [`requests.Response`](https://requests.readthedocs.io/en/latest/api/#requests.Response) object. """ - method = 'DELETE' - return self._request_raw(method, endpoint_path, params=params, headers=headers, data=data, json=json, - cookies=cookies, is_absolute_path=is_absolute_path, files=files, - ignore_auth=ignore_auth, **kwargs) + method = "DELETE" + return self._request_raw( + method, + endpoint_path, + params=params, + headers=headers, + data=data, + json=json, + cookies=cookies, + is_absolute_path=is_absolute_path, + files=files, + ignore_auth=ignore_auth, + **kwargs, + ) @response_error_handling - def delete(self, endpoint_path: Optional[str] = None, params: Dict = None, headers: Dict = None, data: Dict = None, - json: Dict = None, is_absolute_path: bool = False, cookies: Cookie = None, files: Dict = None, - ignore_auth: bool = False, **kwargs) -> requests.Response: + def delete( + self, + endpoint_path: str | None = None, + params: dict | None = None, + headers: dict | None = None, + data: dict | None = None, + json: dict | None = None, + is_absolute_path: bool = False, + cookies: Cookie | None = None, + files: dict | None = None, + ignore_auth: bool = False, + **kwargs, + ) -> requests.Response: """ Constructs a requests DELETE call with specified url and kwargs to process the result. @@ -681,5 +906,15 @@ def delete(self, endpoint_path: Optional[str] = None, params: Dict = None, heade requests.HTTPError: If the API request fails. """ - return self.delete_raw(endpoint_path, params=params, headers=headers, data=data, json=json, cookies=cookies, - is_absolute_path=is_absolute_path, files=files, ignore_auth=ignore_auth, **kwargs) + return self.delete_raw( + endpoint_path, + params=params, + headers=headers, + data=data, + json=json, + cookies=cookies, + is_absolute_path=is_absolute_path, + files=files, + ignore_auth=ignore_auth, + **kwargs, + ) From c76a3c856658404efb4b12423ca0d13050ee45e3 Mon Sep 17 00:00:00 2001 From: soustruh Date: Wed, 1 Oct 2025 18:40:10 +0200 Subject: [PATCH 49/56] =?UTF-8?q?fixed=20import=20(bye=20noqa)=20?= =?UTF-8?q?=F0=9F=91=B7=E2=80=8D=E2=99=82=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/keboola/http_client/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keboola/http_client/http.py b/src/keboola/http_client/http.py index fc81036..ed2d231 100644 --- a/src/keboola/http_client/http.py +++ b/src/keboola/http_client/http.py @@ -5,7 +5,7 @@ import requests from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry # noqa +from urllib3.util import Retry Cookie = dict[str, str] | CookieJar From 251e64b5a95846624af6172da37fa6d72afcaba9 Mon Sep 17 00:00:00 2001 From: soustruh Date: Wed, 1 Oct 2025 18:44:54 +0200 Subject: [PATCH 50/56] misleading variable name --- src/keboola/http_client/async_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 85b2cbd..134ce50 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -55,8 +55,8 @@ def __init__( self.limiter = None if max_requests_per_second: - one_reqeust_per_second_amount = float(1 / max_requests_per_second) - self.limiter = AsyncLimiter(1, one_reqeust_per_second_amount) + max_request_duration = float(1 / max_requests_per_second) + self.limiter = AsyncLimiter(1, max_request_duration) self.default_headers = default_headers or {} self.backoff_factor = backoff_factor From 5c09119ecceddea95b90697a69439c8107417c95 Mon Sep 17 00:00:00 2001 From: soustruh Date: Thu, 2 Oct 2025 22:56:27 +0200 Subject: [PATCH 51/56] consistent headings --- README.md | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ec88d1b..7491294 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,16 @@ - [Quick Start](#quick-start) - [Installation](#installation) - [Structure and Functionality](#structure-and-functionality) - - [`HttpClient`](#httpclient) + - [HttpClient](#httpclient) - [Initialization](#initialization) - - [Default arguments](#default-arguments) - - [Basic authentication](#basic-authentication) - - [Simple POST request](#simple-post-request) - - [Working with URL paths](#working-with-url-paths) - - [Raw request Example](#raw-request-example) - - [Building HTTP client based on HTTPClient Example](#building-http-client-based-on-httpclient-example) - - [Async Usage](#async-usage) - - [Building HTTP client based on AsyncHttpClient Example](#building-http-client-based-on-asynchttpclient-example) + - [Default Arguments](#default-arguments) + - [Basic Authentication](#basic-authentication) + - [Simple POST Request](#simple-post-request) + - [Working with URL Paths](#working-with-url-paths) + - [Raw Request Example](#raw-request-example) + - [Example Client Based on HTTPClient](#example-client-based-on-httpclient) + - [AsyncHttpClient](#asynchttpclient) + - [Example Client Based on AsyncHttpClient](#example-client-based-on-asynchttpclient) # Python HTTP Client @@ -49,7 +49,7 @@ pip install keboola.http-client The package contains a single core module: - `keboola.http_client` - Contains the `HttpClient` class for easy manipulation with APIs and external services -### `HttpClient` +### HttpClient The core class that serves as a tool to communicate with external services. The class is a wrapper around the `requests` library with implemented retry mechanism, and automatic error handling in case of HTTP error returned. @@ -76,7 +76,7 @@ BASE_URL = "https://connection.keboola.com/v2/storage/" cl = HttpClient(BASE_URL) ``` -#### Default arguments +#### Default Arguments For `HttpClient`, it is possible to define default arguments, which will be sent with every request. It's possible to define `default_http_header`, `auth_header` and `default_params` - a default header, a default authentication header and default parameters, respectively. @@ -96,7 +96,7 @@ cl = HttpClient( ) ``` -#### Basic authentication +#### Basic Authentication By specifying the `auth` argument, the `HttpClient` will utilize the basic authentication. @@ -110,7 +110,7 @@ PASSWORD = "@bcd1234" cl = HttpClient(BASE_URL, auth=(USERNAME, PASSWORD)) ``` -#### Simple POST request +#### Simple POST Request Making a simple POST request using `post_raw()` method. @@ -143,7 +143,7 @@ header = {"content-type": "application/json"} response = cl.post(data=data, headers=header) ``` -#### Working with URL paths +#### Working with URL Paths Each of the methods takes an optional positional argument `endpoint_path`. If specified, the value of the `endpoint_path` will be appended to the URL specified in the `base_url` parameter, when initializing the class. When appending the `endpoint_path`, the [`urllib.parse.urljoin()`](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin) function is used. @@ -173,7 +173,7 @@ header = {"token": "token_value"} cl.post_raw("https://anothersite.com/v2", headers=header, is_absolute_path=True) ``` -#### Raw request Example +#### Raw Request Example A simple request made with default authentication header and parameters. @@ -193,7 +193,7 @@ if response.ok is True: print(response.json()) ``` -#### Building HTTP client based on HTTPClient Example +#### Example Client Based on HTTPClient This example demonstrates the default use of the HTTPClient as a base for REST API clients. @@ -225,7 +225,7 @@ cl = KBCStorageClient("my_token") print(cl.get_files()) ``` -## Async Usage +## AsyncHttpClient The package also provides an asynchronous version of the HTTP client called AsyncHttpClient. It allows you to make asynchronous requests using async/await syntax. To use the AsyncHttpClient, import it from keboola.http_client_async: @@ -261,7 +261,8 @@ asyncio.run(main()) The AsyncHttpClient provides initialization and request methods similar to the HttpClient. The request methods return awaitable coroutines that can be awaited in an asynchronous context. -#### Building HTTP client based on AsyncHttpClient Example +#### Example Client Based on AsyncHttpClient + This example demonstrates the default use of the AsyncHttpClient as a base for REST API clients. ```py From 8d8e2f9576bfba02200eb7c10542a590c0b3b8f2 Mon Sep 17 00:00:00 2001 From: soustruh Date: Fri, 3 Oct 2025 00:07:38 +0200 Subject: [PATCH 52/56] =?UTF-8?q?flake8.cfg=20consistent=20with=20cookiecu?= =?UTF-8?q?tter=20template=20+=20minor=20formatting=20fixes=20=F0=9F=8D=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flake8.cfg | 24 ++--- src/keboola/http_client/__init__.py | 2 +- tests/test_async.py | 158 ++++++++++++++++------------ 3 files changed, 96 insertions(+), 88 deletions(-) diff --git a/flake8.cfg b/flake8.cfg index e1352d3..f36c003 100644 --- a/flake8.cfg +++ b/flake8.cfg @@ -1,21 +1,9 @@ [flake8] exclude = - .git, __pycache__, - tests, - mapping.py, - __init__.py -max-line-length = 119 - -# F812: list comprehension redefines ... -# H101: Use TODO(NAME) -# H202: assertRaises Exception too broad -# H233: Python 3.x incompatible use of print operator -# H301: one import per line -# H306: imports not in alphabetical order (time, os) -# H401: docstring should not start with a space -# H403: multi line docstrings should end on a new line -# H404: multi line docstring should start without a leading new line -# H405: multi line docstring summary not separated with an empty line -# H501: Do not use self.__dict__ for string formatting -ignore = F812,H101,H202,H233,H301,H306,H401,H403,H404,H405,H501 \ No newline at end of file + .git, + .venv, + venv, + docs +ignore = E203,W503 +max-line-length = 120 diff --git a/src/keboola/http_client/__init__.py b/src/keboola/http_client/__init__.py index 0ec7a35..b68fe8f 100644 --- a/src/keboola/http_client/__init__.py +++ b/src/keboola/http_client/__init__.py @@ -1,2 +1,2 @@ from .http import HttpClient # noqa -from .async_client import AsyncHttpClient # noqa +from .async_client import AsyncHttpClient # noqa diff --git a/tests/test_async.py b/tests/test_async.py index a72c9d2..3c33779 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -1,6 +1,8 @@ import unittest from unittest.mock import patch + import httpx + from keboola.http_client import AsyncHttpClient @@ -15,7 +17,7 @@ async def test_get(self): client = AsyncHttpClient(self.base_url, retries=self.retries) - with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: + with patch.object(httpx.AsyncClient, "request", return_value=mock_response) as mock_request: response = await client.get("/endpoint") self.assertEqual(response, expected_response) mock_request.assert_called_once_with("GET", url="https://api.example.com/endpoint") @@ -27,11 +29,12 @@ async def test_post(self): client = AsyncHttpClient(self.base_url, retries=self.retries) - with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: + with patch.object(httpx.AsyncClient, "request", return_value=mock_response) as mock_request: response = await client.post("/endpoint", json={"data": "example"}) self.assertEqual(response, expected_response) - mock_request.assert_called_once_with("POST", url="https://api.example.com/endpoint", - json={"data": "example"}) + mock_request.assert_called_once_with( + "POST", url="https://api.example.com/endpoint", json={"data": "example"} + ) async def test_handle_success_response(self): expected_response = {"message": "Success"} @@ -40,7 +43,7 @@ async def test_handle_success_response(self): client = AsyncHttpClient(self.base_url, retries=self.retries) - with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: + with patch.object(httpx.AsyncClient, "request", return_value=mock_response) as mock_request: response = await client.get("/endpoint") self.assertEqual(response, expected_response) mock_request.assert_called_once_with("GET", url="https://api.example.com/endpoint") @@ -51,7 +54,7 @@ async def test_handle_client_error_response(self): client = AsyncHttpClient(self.base_url, retries=self.retries, retry_status_codes=[404]) - with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: + with patch.object(httpx.AsyncClient, "request", return_value=mock_response) as mock_request: with self.assertRaises(httpx.HTTPStatusError): await client.get("/endpoint") @@ -65,7 +68,7 @@ async def test_handle_server_error_response(self): client = AsyncHttpClient(self.base_url, retries=self.retries, retry_status_codes=[500]) - with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: + with patch.object(httpx.AsyncClient, "request", return_value=mock_response) as mock_request: with self.assertRaises(httpx.HTTPStatusError): await client.get("/endpoint") @@ -73,7 +76,7 @@ async def test_handle_server_error_response(self): mock_request.assert_called_with("GET", url="https://api.example.com/endpoint") - @patch.object(httpx.AsyncClient, 'request') + @patch.object(httpx.AsyncClient, "request") async def test_post_raw_default_pars_with_none_custom_pars_passes(self, mock_request): url = f"{self.base_url}/endpoint" test_def_par = {"default_par": "test"} @@ -84,7 +87,7 @@ async def test_post_raw_default_pars_with_none_custom_pars_passes(self, mock_req mock_request.assert_called_once_with("POST", url=url, params=test_def_par) - @patch.object(httpx.AsyncClient, 'request') + @patch.object(httpx.AsyncClient, "request") async def test_post_default_pars_with_none_custom_pars_passes(self, mock_request): url = f"{self.base_url}/endpoint" test_def_par = {"default_par": "test"} @@ -95,7 +98,7 @@ async def test_post_default_pars_with_none_custom_pars_passes(self, mock_request mock_request.assert_called_once_with("POST", url=url, params=test_def_par) - @patch.object(httpx.AsyncClient, 'request') + @patch.object(httpx.AsyncClient, "request") async def test_post_raw_default_pars_with_custom_pars_passes(self, mock_request): url = f"{self.base_url}/endpoint" test_def_par = {"default_par": "test"} @@ -108,7 +111,7 @@ async def test_post_raw_default_pars_with_custom_pars_passes(self, mock_request) test_cust_def_par = {**test_def_par, **cust_par} mock_request.assert_called_once_with("POST", url=url, params=test_cust_def_par) - @patch.object(httpx.AsyncClient, 'request') + @patch.object(httpx.AsyncClient, "request") async def test_post_default_pars_with_custom_pars_passes(self, mock_request): url = f"{self.base_url}/endpoint" test_def_par = {"default_par": "test"} @@ -121,7 +124,7 @@ async def test_post_default_pars_with_custom_pars_passes(self, mock_request): test_cust_def_par = {**test_def_par, **cust_par} mock_request.assert_called_once_with("POST", url=url, params=test_cust_def_par) - @patch.object(httpx.AsyncClient, 'request') + @patch.object(httpx.AsyncClient, "request") async def test_post_raw_default_pars_with_custom_pars_to_None_passes(self, mock_request): url = f"{self.base_url}/endpoint" test_def_par = {"default_par": "test"} @@ -136,7 +139,7 @@ async def test_post_raw_default_pars_with_custom_pars_to_None_passes(self, mock_ test_cust_def_par = {**test_def_par, **_cust_par_transformed} mock_request.assert_called_once_with("POST", url=url, params=test_cust_def_par) - @patch.object(httpx.AsyncClient, 'request') + @patch.object(httpx.AsyncClient, "request") async def test_post_default_pars_with_custom_pars_to_None_passes(self, mock_request): url = f"{self.base_url}/endpoint" test_def_par = {"default_par": "test"} @@ -151,7 +154,7 @@ async def test_post_default_pars_with_custom_pars_to_None_passes(self, mock_requ test_cust_def_par = {**test_def_par, **_cust_par_transformed} mock_request.assert_called_once_with("POST", url=url, params=test_cust_def_par) - @patch.object(httpx.AsyncClient, 'request') + @patch.object(httpx.AsyncClient, "request") async def test_all_methods_requests_raw_with_custom_pars_passes(self, mock_request): client = AsyncHttpClient(self.base_url) @@ -159,138 +162,154 @@ async def test_all_methods_requests_raw_with_custom_pars_passes(self, mock_reque for m in client.ALLOWED_METHODS: await client._request(m, ignore_auth=False, params=cust_par) - mock_request.assert_called_with(m, url=self.base_url+"/", params=cust_par) + mock_request.assert_called_with(m, url=self.base_url + "/", params=cust_par) - @patch.object(httpx.AsyncClient, 'request') + @patch.object(httpx.AsyncClient, "request") async def test_all_methods_skip_auth(self, mock_request): client = AsyncHttpClient(self.base_url, auth=("my_user", "password123")) - for m in ['GET', 'POST', 'PATCH', 'UPDATE', 'PUT', 'DELETE']: + for m in ["GET", "POST", "PATCH", "UPDATE", "PUT", "DELETE"]: await client._request(m, ignore_auth=True) - mock_request.assert_called_with(m, url=self.base_url+"/") + mock_request.assert_called_with(m, url=self.base_url + "/") - @patch.object(httpx.AsyncClient, 'request') + @patch.object(httpx.AsyncClient, "request") async def test_request_skip_auth_header(self, mock_request): def_header = {"def_header": "test"} - client = AsyncHttpClient('http://example.com', default_headers=def_header, - auth_header={"Authorization": "test"}) + client = AsyncHttpClient( + "http://example.com", default_headers=def_header, auth_header={"Authorization": "test"} + ) - await client._request('POST', 'abc', ignore_auth=True) - mock_request.assert_called_with('POST', url="http://example.com/abc", headers=def_header) + await client._request("POST", "abc", ignore_auth=True) + mock_request.assert_called_with("POST", url="http://example.com/abc", headers=def_header) - @patch.object(httpx.AsyncClient, 'request') + @patch.object(httpx.AsyncClient, "request") async def test_request_auth(self, mock_request): def_header = {"def_header": "test"} auth = ("my_user", "password123") client = AsyncHttpClient(self.base_url, auth=auth, default_headers=def_header) - await client._request('POST', 'abc') - mock_request.assert_called_with('POST', url=self.base_url+"/abc", headers=def_header, - auth=auth) + await client._request("POST", "abc") + mock_request.assert_called_with("POST", url=self.base_url + "/abc", headers=def_header, auth=auth) - @patch.object(httpx.AsyncClient, 'request') + @patch.object(httpx.AsyncClient, "request") async def test_all_methods(self, mock_request): - client = AsyncHttpClient(self.base_url, default_headers={'header1': 'headerval'}, - auth_header={'api_token': 'abdc1234'}) + client = AsyncHttpClient( + self.base_url, default_headers={"header1": "headerval"}, auth_header={"api_token": "abdc1234"} + ) - target_url = f'{self.base_url}/abc' + target_url = f"{self.base_url}/abc" for m in client.ALLOWED_METHODS: - await client._request(m, 'abc', params={'exclude': 'componentDetails'}, headers={'abc': '123'}, - data={'attr1': 'val1'}) - mock_request.assert_called_with(m, url=target_url, - params={'exclude': 'componentDetails'}, - headers={'api_token': 'abdc1234', 'header1': 'headerval', 'abc': '123'}, - data={'attr1': 'val1'}) - - @patch.object(httpx.AsyncClient, 'request') + await client._request( + m, "abc", params={"exclude": "componentDetails"}, headers={"abc": "123"}, data={"attr1": "val1"} + ) + mock_request.assert_called_with( + m, + url=target_url, + params={"exclude": "componentDetails"}, + headers={"api_token": "abdc1234", "header1": "headerval", "abc": "123"}, + data={"attr1": "val1"}, + ) + + @patch.object(httpx.AsyncClient, "request") async def test_all_methods_requests_raw_with_is_absolute_path_true(self, mock_request): def_header = {"def_header": "test"} client = AsyncHttpClient(self.base_url, default_headers=def_header) for m in client.ALLOWED_METHODS: - await client._request(m, 'http://example2.com/v1/', is_absolute_path=True) - mock_request.assert_called_with(m, url='http://example2.com/v1/', headers=def_header) + await client._request(m, "http://example2.com/v1/", is_absolute_path=True) + mock_request.assert_called_with(m, url="http://example2.com/v1/", headers=def_header) - @patch.object(httpx.AsyncClient, 'request') + @patch.object(httpx.AsyncClient, "request") async def test_all_methods_requests_raw_with_is_absolute_path_false(self, mock_request): def_header = {"def_header": "test"} client = AsyncHttpClient(self.base_url, default_headers=def_header) for m in client.ALLOWED_METHODS: - await client._request(m, 'cars') - mock_request.assert_called_with(m, url=self.base_url+"/cars", headers=def_header) + await client._request(m, "cars") + mock_request.assert_called_with(m, url=self.base_url + "/cars", headers=def_header) - @patch.object(httpx.AsyncClient, 'request') + @patch.object(httpx.AsyncClient, "request") async def test_all_methods_kwargs(self, mock_request): client = AsyncHttpClient(self.base_url) for m in client.ALLOWED_METHODS: - await client._request(m, 'cars', data={'data': '123'}, cert='/path/to/cert', files={'a': '/path/to/file'}, - params={'par1': 'val1'}) - - mock_request.assert_called_with(m, url=self.base_url+"/cars", data={'data': '123'}, - cert='/path/to/cert', files={'a': '/path/to/file'}, - params={'par1': 'val1'}) + await client._request( + m, + "cars", + data={"data": "123"}, + cert="/path/to/cert", + files={"a": "/path/to/file"}, + params={"par1": "val1"}, + ) + + mock_request.assert_called_with( + m, + url=self.base_url + "/cars", + data={"data": "123"}, + cert="/path/to/cert", + files={"a": "/path/to/file"}, + params={"par1": "val1"}, + ) async def test_build_url_rel_path(self): - url = 'https://example.com/' + url = "https://example.com/" cl = AsyncHttpClient(url) - expected_url = 'https://example.com/storage' - actual_url = await cl._build_url('storage') + expected_url = "https://example.com/storage" + actual_url = await cl._build_url("storage") self.assertEqual(expected_url, actual_url) async def test_build_url_abs_path(self): - url = 'https://example.com/' + url = "https://example.com/" cl = AsyncHttpClient(url) - expected_url = 'https://example2.com/storage' - actual_url = await cl._build_url('https://example2.com/storage', True) + expected_url = "https://example2.com/storage" + actual_url = await cl._build_url("https://example2.com/storage", True) self.assertEqual(expected_url, actual_url) async def test_build_url_empty_endpoint_path_leads_to_base_url(self): - url = 'https://example.com/' + url = "https://example.com/" cl = AsyncHttpClient(url) expected_url = url actual_url = await cl._build_url() self.assertEqual(expected_url, actual_url) - actual_url = await cl._build_url('') + actual_url = await cl._build_url("") self.assertEqual(expected_url, actual_url) actual_url = await cl._build_url(None) self.assertEqual(expected_url, actual_url) - actual_url = await cl._build_url('', is_absolute_path=True) + actual_url = await cl._build_url("", is_absolute_path=True) self.assertEqual(expected_url, actual_url) actual_url = await cl._build_url(None, is_absolute_path=True) self.assertEqual(expected_url, actual_url) async def test_build_url_base_url_appends_slash(self): - url = 'https://example.com' + url = "https://example.com" cl = AsyncHttpClient(url) - expected_base_url = 'https://example.com/' + expected_base_url = "https://example.com/" self.assertEqual(expected_base_url, cl.base_url) async def test_update_auth_header_None(self): existing_header = None - new_header = {'api_token': 'token_value'} + new_header = {"api_token": "token_value"} - cl = AsyncHttpClient('https://example.com', auth_header=existing_header) + cl = AsyncHttpClient("https://example.com", auth_header=existing_header) await cl.update_auth_header(new_header, overwrite=False) self.assertDictEqual(cl._auth_header, new_header) - new_header_2 = {'password': '123'} + new_header_2 = {"password": "123"} await cl.update_auth_header(new_header_2, overwrite=True) self.assertDictEqual(cl._auth_header, new_header_2) async def test_update_existing_auth_header(self): - existing_header = {'authorization': 'value'} - new_header = {'api_token': 'token_value'} + existing_header = {"authorization": "value"} + new_header = {"api_token": "token_value"} - cl = AsyncHttpClient('https://example.com', auth_header=existing_header) + cl = AsyncHttpClient("https://example.com", auth_header=existing_header) await cl.update_auth_header(new_header, overwrite=False) self.assertDictEqual(cl._auth_header, {**existing_header, **new_header}) @@ -300,11 +319,12 @@ async def test_detailed_exception(self): client = AsyncHttpClient(self.base_url) - with patch.object(httpx.AsyncClient, 'request', return_value=mock_response) as mock_request: + with patch.object(httpx.AsyncClient, "request", return_value=mock_response) as _: with self.assertRaises(httpx.HTTPStatusError) as e: await client.get("/endpoint") assert "Client error '404 Not Found' for url 'https://api.example.com/endpoint'" in str(e.exception) + if __name__ == "__main__": unittest.main() From 841709fb850b8a582b4f1e66532582c91dbb41e9 Mon Sep 17 00:00:00 2001 From: soustruh Date: Fri, 3 Oct 2025 00:07:53 +0200 Subject: [PATCH 53/56] =?UTF-8?q?uv=20=F0=9F=92=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy.yml | 47 +- .github/workflows/deploy_to_test.yml | 48 +- .github/workflows/push_dev.yml | 34 +- LICENSE | 2 +- pyproject.toml | 50 +++ requirements.txt | 3 - setup.py | 44 -- uv.lock | 640 +++++++++++++++++++++++++++ 8 files changed, 766 insertions(+), 102 deletions(-) create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.py create mode 100644 uv.lock diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e0faf35..a1d90fe 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,4 +1,4 @@ -name: Build & Upload Python Package to PYPI production +name: Build & Upload Python Package to PyPI on: release: @@ -10,28 +10,35 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 + - name: Checkout 🛒 + uses: actions/checkout@v4 + + - name: Install uv 💜 + uses: astral-sh/setup-uv@v6 + + - name: Install and run ruff 🐶 + uses: astral-sh/ruff-action@v3 + + - name: Set up Python 🐍 + uses: actions/setup-python@v4 with: - python-version: '3.12' - - name: Install dependencies + python-version: "3.13" + + - name: Install dependencies 📦 run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - pip install flake8 pytest - pip install -r requirements.txt - - name: Lint with flake8 + uv sync --all-groups --frozen + + - name: Lint with flake8 ❄️ run: | - # stop the build if there are Python syntax errors or undefined names - flake8 src/ --config=flake8.cfg - - name: Test with pytest + uv run flake8 --config flake8.cfg + + - name: Test with pytest ✅ run: | - pytest tests - - name: Build and publish + uv run pytest tests + + - name: Build and publish 🚀 env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + UV_PUBLISH_TOKEN: ${{ secrets.PYPI_PASSWORD }} run: | - python setup.py sdist bdist_wheel - twine upload dist/* \ No newline at end of file + uv build + uv publish diff --git a/.github/workflows/deploy_to_test.yml b/.github/workflows/deploy_to_test.yml index 0472b35..c4e3d95 100644 --- a/.github/workflows/deploy_to_test.yml +++ b/.github/workflows/deploy_to_test.yml @@ -1,4 +1,4 @@ -name: Build & Deploy Python Package To Test PYPI +name: Build & Upload Python Package To Test PyPI on: create: @@ -6,33 +6,39 @@ on: - 0.*a - 1.*a - jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 + - name: Checkout 🛒 + uses: actions/checkout@v4 + + - name: Install uv 💜 + uses: astral-sh/setup-uv@v6 + + - name: Install and run ruff 🐶 + uses: astral-sh/ruff-action@v3 + + - name: Set up Python 🐍 + uses: actions/setup-python@v4 with: - python-version: '3.12' - - name: Install dependencies + python-version: "3.13" + + - name: Install dependencies 📦 run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - pip install flake8 pytest - pip install -r requirements.txt - - name: Lint with flake8 + uv sync --all-groups --frozen + + - name: Lint with flake8 ❄️ run: | - # stop the build if there are Python syntax errors or undefined names - flake8 src/ --config=flake8.cfg - - name: Test with pytest + uv run flake8 --config flake8.cfg + + - name: Test with pytest ✅ run: | - pytest tests - - name: Build and publish + uv run pytest tests + + - name: Build and publish 🚀 env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME_TEST }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD_TEST }} + UV_PUBLISH_TOKEN: ${{ secrets.PYPI_PASSWORD_TEST }} run: | - python setup.py sdist bdist_wheel - twine upload --repository testpypi dist/* \ No newline at end of file + uv build + uv publish --index testpypi diff --git a/.github/workflows/push_dev.yml b/.github/workflows/push_dev.yml index 71cf2e7..e2dd037 100644 --- a/.github/workflows/push_dev.yml +++ b/.github/workflows/push_dev.yml @@ -8,23 +8,31 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ 3.12 ] + python-version: [ 3.8, 3.13 ] steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - name: Checkout 🛒 + uses: actions/checkout@v4 + + - name: Install uv 💜 + uses: astral-sh/setup-uv@v6 + + - name: Install and run ruff 🐶 + uses: astral-sh/ruff-action@v3 + + - name: Set up Python ${{ matrix.python-version }} 🐍 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + + - name: Install dependencies 📦 run: | - python -m pip install --upgrade pip - pip install flake8 pytest - pip install -r requirements.txt - - name: Lint with flake8 + uv sync --all-groups --frozen + + - name: Lint with flake8 ❄️ run: | - # stop the build if there are Python syntax errors or undefined names - flake8 src/ --config=flake8.cfg - - name: Test with pytest + uv run flake8 --config flake8.cfg + + - name: Test with pytest ✅ run: | - pytest tests \ No newline at end of file + uv run pytest tests diff --git a/LICENSE b/LICENSE index acfe65d..db8be01 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 Keboola +Copyright (c) 2025 Keboola Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6e4b095 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,50 @@ +[project] +name = "keboola.http-client" +version = "1.1.5" +dependencies = [ + "aiolimiter>=1.2.1", + "httpx>=0.28.1", + "requests>=2.32.4", +] +requires-python = ">=3.8" + +authors = [ + { name = "Keboola KDS Team", email = "support@keboola.com" } +] +description = "General HTTP requests library for Python applications running in Keboola Connection environment" +readme = "README.md" +license = "MIT" +license-files = [ "LICENSE" ] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Operating System :: OS Independent", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Education", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + "Development Status :: 4 - Beta", +] + +[project.urls] +Documentation = "https://htmlpreview.github.io/?https://raw.githubusercontent.com/keboola/python-http-client/main/docs/api-html/http_client/http.html" +Repository = "https://github.com/keboola/python-http-client" + +[dependency-groups] +dev = [ + "flake8>=5.0.4", + "pytest>=8.3.5", + "ruff>=0.13.2", +] + +[[tool.uv.index]] +name = "testpypi" +url = "https://test.pypi.org/simple/" +publish-url = "https://test.pypi.org/legacy/" +explicit = true diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index af4b5ac..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -requests -httpx==0.27.0 -aiolimiter==1.1.0 diff --git a/setup.py b/setup.py deleted file mode 100644 index 7999ea4..0000000 --- a/setup.py +++ /dev/null @@ -1,44 +0,0 @@ -import setuptools - -with open("README.md", "r") as fh: - long_description = fh.read() - -project_urls = { - 'Documentation': 'https://htmlpreview.github.io/?https://raw.githubusercontent.com/keboola/' - 'python-http-client/main/docs/api-html/http_client/http.html' -} - -setuptools.setup( - name="keboola.http_client", - version="1.0.0", - author="Keboola KDS Team", - project_urls=project_urls, - setup_requires=['pytest-runner', 'flake8'], - tests_require=['pytest'], - install_requires=[ - 'requests', - 'httpx' - ], - author_email="support@keboola.com", - description="General HTTP requests library for Python applications running in Keboola Connection environment", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/keboola/python-http-client", - packages=['keboola.http_client'], - package_dir={'': 'src'}, - include_package_data=True, - zip_safe=False, - test_suite='tests', - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Education", - "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules", - "Development Status :: 4 - Beta" - ], - python_requires='>=3.7' -) diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..c67ec64 --- /dev/null +++ b/uv.lock @@ -0,0 +1,640 @@ +version = 1 +revision = 3 +requires-python = ">=3.8" +resolution-markers = [ + "python_full_version >= '3.9'", + "python_full_version >= '3.8.1' and python_full_version < '3.9'", + "python_full_version < '3.8.1'", +] + +[[package]] +name = "aiolimiter" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/23/b52debf471f7a1e42e362d959a3982bdcb4fe13a5d46e63d28868807a79c/aiolimiter-1.2.1.tar.gz", hash = "sha256:e02a37ea1a855d9e832252a105420ad4d15011505512a1a1d814647451b5cca9", size = 7185, upload-time = "2024-12-08T15:31:51.496Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/ba/df6e8e1045aebc4778d19b8a3a9bc1808adb1619ba94ca354d9ba17d86c3/aiolimiter-1.2.1-py3-none-any.whl", hash = "sha256:d3f249e9059a20badcb56b61601a83556133655c11d1eb3dd3e04ff069e5f3c7", size = 6711, upload-time = "2024-12-08T15:31:49.874Z" }, +] + +[[package]] +name = "anyio" +version = "4.5.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.8.1' and python_full_version < '3.9'", + "python_full_version < '3.8.1'", +] +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.9'" }, + { name = "idna", marker = "python_full_version < '3.9'" }, + { name = "sniffio", marker = "python_full_version < '3.9'" }, + { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/f9/9a7ce600ebe7804daf90d4d48b1c0510a4561ddce43a596be46676f82343/anyio-4.5.2.tar.gz", hash = "sha256:23009af4ed04ce05991845451e11ef02fc7c5ed29179ac9a420e5ad0ac7ddc5b", size = 171293, upload-time = "2024-10-13T22:18:03.307Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/b4/f7e396030e3b11394436358ca258a81d6010106582422f23443c16ca1873/anyio-4.5.2-py3-none-any.whl", hash = "sha256:c011ee36bc1e8ba40e5a81cb9df91925c218fe9b778554e0b56a21e1b5d4716f", size = 89766, upload-time = "2024-10-13T22:18:01.524Z" }, +] + +[[package]] +name = "anyio" +version = "4.11.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.9'", +] +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version >= '3.9' and python_full_version < '3.11'" }, + { name = "idna", marker = "python_full_version >= '3.9'" }, + { name = "sniffio", marker = "python_full_version >= '3.9'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9' and python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/78/7d432127c41b50bccba979505f272c16cbcadcc33645d5fa3a738110ae75/anyio-4.11.0.tar.gz", hash = "sha256:82a8d0b81e318cc5ce71a5f1f8b5c4e63619620b63141ef8c995fa0db95a57c4", size = 219094, upload-time = "2025-09-23T09:19:12.58Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc", size = 109097, upload-time = "2025-09-23T09:19:10.601Z" }, +] + +[[package]] +name = "certifi" +version = "2025.8.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/98/f3b8013223728a99b908c9344da3aa04ee6e3fa235f19409033eda92fb78/charset_normalizer-3.4.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72", size = 207695, upload-time = "2025-08-09T07:55:36.452Z" }, + { url = "https://files.pythonhosted.org/packages/21/40/5188be1e3118c82dcb7c2a5ba101b783822cfb413a0268ed3be0468532de/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe", size = 147153, upload-time = "2025-08-09T07:55:38.467Z" }, + { url = "https://files.pythonhosted.org/packages/37/60/5d0d74bc1e1380f0b72c327948d9c2aca14b46a9efd87604e724260f384c/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601", size = 160428, upload-time = "2025-08-09T07:55:40.072Z" }, + { url = "https://files.pythonhosted.org/packages/85/9a/d891f63722d9158688de58d050c59dc3da560ea7f04f4c53e769de5140f5/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c", size = 157627, upload-time = "2025-08-09T07:55:41.706Z" }, + { url = "https://files.pythonhosted.org/packages/65/1a/7425c952944a6521a9cfa7e675343f83fd82085b8af2b1373a2409c683dc/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2", size = 152388, upload-time = "2025-08-09T07:55:43.262Z" }, + { url = "https://files.pythonhosted.org/packages/f0/c9/a2c9c2a355a8594ce2446085e2ec97fd44d323c684ff32042e2a6b718e1d/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0", size = 150077, upload-time = "2025-08-09T07:55:44.903Z" }, + { url = "https://files.pythonhosted.org/packages/3b/38/20a1f44e4851aa1c9105d6e7110c9d020e093dfa5836d712a5f074a12bf7/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0", size = 161631, upload-time = "2025-08-09T07:55:46.346Z" }, + { url = "https://files.pythonhosted.org/packages/a4/fa/384d2c0f57edad03d7bec3ebefb462090d8905b4ff5a2d2525f3bb711fac/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0", size = 159210, upload-time = "2025-08-09T07:55:47.539Z" }, + { url = "https://files.pythonhosted.org/packages/33/9e/eca49d35867ca2db336b6ca27617deed4653b97ebf45dfc21311ce473c37/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a", size = 153739, upload-time = "2025-08-09T07:55:48.744Z" }, + { url = "https://files.pythonhosted.org/packages/2a/91/26c3036e62dfe8de8061182d33be5025e2424002125c9500faff74a6735e/charset_normalizer-3.4.3-cp310-cp310-win32.whl", hash = "sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f", size = 99825, upload-time = "2025-08-09T07:55:50.305Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c6/f05db471f81af1fa01839d44ae2a8bfeec8d2a8b4590f16c4e7393afd323/charset_normalizer-3.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669", size = 107452, upload-time = "2025-08-09T07:55:51.461Z" }, + { url = "https://files.pythonhosted.org/packages/7f/b5/991245018615474a60965a7c9cd2b4efbaabd16d582a5547c47ee1c7730b/charset_normalizer-3.4.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b", size = 204483, upload-time = "2025-08-09T07:55:53.12Z" }, + { url = "https://files.pythonhosted.org/packages/c7/2a/ae245c41c06299ec18262825c1569c5d3298fc920e4ddf56ab011b417efd/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64", size = 145520, upload-time = "2025-08-09T07:55:54.712Z" }, + { url = "https://files.pythonhosted.org/packages/3a/a4/b3b6c76e7a635748c4421d2b92c7b8f90a432f98bda5082049af37ffc8e3/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91", size = 158876, upload-time = "2025-08-09T07:55:56.024Z" }, + { url = "https://files.pythonhosted.org/packages/e2/e6/63bb0e10f90a8243c5def74b5b105b3bbbfb3e7bb753915fe333fb0c11ea/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f", size = 156083, upload-time = "2025-08-09T07:55:57.582Z" }, + { url = "https://files.pythonhosted.org/packages/87/df/b7737ff046c974b183ea9aa111b74185ac8c3a326c6262d413bd5a1b8c69/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07", size = 150295, upload-time = "2025-08-09T07:55:59.147Z" }, + { url = "https://files.pythonhosted.org/packages/61/f1/190d9977e0084d3f1dc169acd060d479bbbc71b90bf3e7bf7b9927dec3eb/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30", size = 148379, upload-time = "2025-08-09T07:56:00.364Z" }, + { url = "https://files.pythonhosted.org/packages/4c/92/27dbe365d34c68cfe0ca76f1edd70e8705d82b378cb54ebbaeabc2e3029d/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14", size = 160018, upload-time = "2025-08-09T07:56:01.678Z" }, + { url = "https://files.pythonhosted.org/packages/99/04/baae2a1ea1893a01635d475b9261c889a18fd48393634b6270827869fa34/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c", size = 157430, upload-time = "2025-08-09T07:56:02.87Z" }, + { url = "https://files.pythonhosted.org/packages/2f/36/77da9c6a328c54d17b960c89eccacfab8271fdaaa228305330915b88afa9/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae", size = 151600, upload-time = "2025-08-09T07:56:04.089Z" }, + { url = "https://files.pythonhosted.org/packages/64/d4/9eb4ff2c167edbbf08cdd28e19078bf195762e9bd63371689cab5ecd3d0d/charset_normalizer-3.4.3-cp311-cp311-win32.whl", hash = "sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849", size = 99616, upload-time = "2025-08-09T07:56:05.658Z" }, + { url = "https://files.pythonhosted.org/packages/f4/9c/996a4a028222e7761a96634d1820de8a744ff4327a00ada9c8942033089b/charset_normalizer-3.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c", size = 107108, upload-time = "2025-08-09T07:56:07.176Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" }, + { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" }, + { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104, upload-time = "2025-08-09T07:56:13.014Z" }, + { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830, upload-time = "2025-08-09T07:56:14.428Z" }, + { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854, upload-time = "2025-08-09T07:56:16.051Z" }, + { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670, upload-time = "2025-08-09T07:56:17.314Z" }, + { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501, upload-time = "2025-08-09T07:56:18.641Z" }, + { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173, upload-time = "2025-08-09T07:56:20.289Z" }, + { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822, upload-time = "2025-08-09T07:56:21.551Z" }, + { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543, upload-time = "2025-08-09T07:56:23.115Z" }, + { url = "https://files.pythonhosted.org/packages/65/ca/2135ac97709b400c7654b4b764daf5c5567c2da45a30cdd20f9eefe2d658/charset_normalizer-3.4.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", size = 205326, upload-time = "2025-08-09T07:56:24.721Z" }, + { url = "https://files.pythonhosted.org/packages/71/11/98a04c3c97dd34e49c7d247083af03645ca3730809a5509443f3c37f7c99/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", size = 146008, upload-time = "2025-08-09T07:56:26.004Z" }, + { url = "https://files.pythonhosted.org/packages/60/f5/4659a4cb3c4ec146bec80c32d8bb16033752574c20b1252ee842a95d1a1e/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", size = 159196, upload-time = "2025-08-09T07:56:27.25Z" }, + { url = "https://files.pythonhosted.org/packages/86/9e/f552f7a00611f168b9a5865a1414179b2c6de8235a4fa40189f6f79a1753/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", size = 156819, upload-time = "2025-08-09T07:56:28.515Z" }, + { url = "https://files.pythonhosted.org/packages/7e/95/42aa2156235cbc8fa61208aded06ef46111c4d3f0de233107b3f38631803/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", size = 151350, upload-time = "2025-08-09T07:56:29.716Z" }, + { url = "https://files.pythonhosted.org/packages/c2/a9/3865b02c56f300a6f94fc631ef54f0a8a29da74fb45a773dfd3dcd380af7/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", size = 148644, upload-time = "2025-08-09T07:56:30.984Z" }, + { url = "https://files.pythonhosted.org/packages/77/d9/cbcf1a2a5c7d7856f11e7ac2d782aec12bdfea60d104e60e0aa1c97849dc/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9", size = 160468, upload-time = "2025-08-09T07:56:32.252Z" }, + { url = "https://files.pythonhosted.org/packages/f6/42/6f45efee8697b89fda4d50580f292b8f7f9306cb2971d4b53f8914e4d890/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", size = 158187, upload-time = "2025-08-09T07:56:33.481Z" }, + { url = "https://files.pythonhosted.org/packages/70/99/f1c3bdcfaa9c45b3ce96f70b14f070411366fa19549c1d4832c935d8e2c3/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", size = 152699, upload-time = "2025-08-09T07:56:34.739Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ad/b0081f2f99a4b194bcbb1934ef3b12aa4d9702ced80a37026b7607c72e58/charset_normalizer-3.4.3-cp313-cp313-win32.whl", hash = "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", size = 99580, upload-time = "2025-08-09T07:56:35.981Z" }, + { url = "https://files.pythonhosted.org/packages/9a/8f/ae790790c7b64f925e5c953b924aaa42a243fb778fed9e41f147b2a5715a/charset_normalizer-3.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", size = 107366, upload-time = "2025-08-09T07:56:37.339Z" }, + { url = "https://files.pythonhosted.org/packages/8e/91/b5a06ad970ddc7a0e513112d40113e834638f4ca1120eb727a249fb2715e/charset_normalizer-3.4.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", size = 204342, upload-time = "2025-08-09T07:56:38.687Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ec/1edc30a377f0a02689342f214455c3f6c2fbedd896a1d2f856c002fc3062/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", size = 145995, upload-time = "2025-08-09T07:56:40.048Z" }, + { url = "https://files.pythonhosted.org/packages/17/e5/5e67ab85e6d22b04641acb5399c8684f4d37caf7558a53859f0283a650e9/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", size = 158640, upload-time = "2025-08-09T07:56:41.311Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e5/38421987f6c697ee3722981289d554957c4be652f963d71c5e46a262e135/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", size = 156636, upload-time = "2025-08-09T07:56:43.195Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e4/5a075de8daa3ec0745a9a3b54467e0c2967daaaf2cec04c845f73493e9a1/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", size = 150939, upload-time = "2025-08-09T07:56:44.819Z" }, + { url = "https://files.pythonhosted.org/packages/02/f7/3611b32318b30974131db62b4043f335861d4d9b49adc6d57c1149cc49d4/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", size = 148580, upload-time = "2025-08-09T07:56:46.684Z" }, + { url = "https://files.pythonhosted.org/packages/7e/61/19b36f4bd67f2793ab6a99b979b4e4f3d8fc754cbdffb805335df4337126/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", size = 159870, upload-time = "2025-08-09T07:56:47.941Z" }, + { url = "https://files.pythonhosted.org/packages/06/57/84722eefdd338c04cf3030ada66889298eaedf3e7a30a624201e0cbe424a/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", size = 157797, upload-time = "2025-08-09T07:56:49.756Z" }, + { url = "https://files.pythonhosted.org/packages/72/2a/aff5dd112b2f14bcc3462c312dce5445806bfc8ab3a7328555da95330e4b/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", size = 152224, upload-time = "2025-08-09T07:56:51.369Z" }, + { url = "https://files.pythonhosted.org/packages/b7/8c/9839225320046ed279c6e839d51f028342eb77c91c89b8ef2549f951f3ec/charset_normalizer-3.4.3-cp314-cp314-win32.whl", hash = "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", size = 100086, upload-time = "2025-08-09T07:56:52.722Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7a/36fbcf646e41f710ce0a563c1c9a343c6edf9be80786edeb15b6f62e17db/charset_normalizer-3.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", size = 107400, upload-time = "2025-08-09T07:56:55.172Z" }, + { url = "https://files.pythonhosted.org/packages/22/82/63a45bfc36f73efe46731a3a71cb84e2112f7e0b049507025ce477f0f052/charset_normalizer-3.4.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0f2be7e0cf7754b9a30eb01f4295cc3d4358a479843b31f328afd210e2c7598c", size = 198805, upload-time = "2025-08-09T07:56:56.496Z" }, + { url = "https://files.pythonhosted.org/packages/0c/52/8b0c6c3e53f7e546a5e49b9edb876f379725914e1130297f3b423c7b71c5/charset_normalizer-3.4.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c60e092517a73c632ec38e290eba714e9627abe9d301c8c8a12ec32c314a2a4b", size = 142862, upload-time = "2025-08-09T07:56:57.751Z" }, + { url = "https://files.pythonhosted.org/packages/59/c0/a74f3bd167d311365e7973990243f32c35e7a94e45103125275b9e6c479f/charset_normalizer-3.4.3-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:252098c8c7a873e17dd696ed98bbe91dbacd571da4b87df3736768efa7a792e4", size = 155104, upload-time = "2025-08-09T07:56:58.984Z" }, + { url = "https://files.pythonhosted.org/packages/1a/79/ae516e678d6e32df2e7e740a7be51dc80b700e2697cb70054a0f1ac2c955/charset_normalizer-3.4.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3653fad4fe3ed447a596ae8638b437f827234f01a8cd801842e43f3d0a6b281b", size = 152598, upload-time = "2025-08-09T07:57:00.201Z" }, + { url = "https://files.pythonhosted.org/packages/00/bd/ef9c88464b126fa176f4ef4a317ad9b6f4d30b2cffbc43386062367c3e2c/charset_normalizer-3.4.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8999f965f922ae054125286faf9f11bc6932184b93011d138925a1773830bbe9", size = 147391, upload-time = "2025-08-09T07:57:01.441Z" }, + { url = "https://files.pythonhosted.org/packages/7a/03/cbb6fac9d3e57f7e07ce062712ee80d80a5ab46614684078461917426279/charset_normalizer-3.4.3-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d95bfb53c211b57198bb91c46dd5a2d8018b3af446583aab40074bf7988401cb", size = 145037, upload-time = "2025-08-09T07:57:02.638Z" }, + { url = "https://files.pythonhosted.org/packages/64/d1/f9d141c893ef5d4243bc75c130e95af8fd4bc355beff06e9b1e941daad6e/charset_normalizer-3.4.3-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:5b413b0b1bfd94dbf4023ad6945889f374cd24e3f62de58d6bb102c4d9ae534a", size = 156425, upload-time = "2025-08-09T07:57:03.898Z" }, + { url = "https://files.pythonhosted.org/packages/c5/35/9c99739250742375167bc1b1319cd1cec2bf67438a70d84b2e1ec4c9daa3/charset_normalizer-3.4.3-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:b5e3b2d152e74e100a9e9573837aba24aab611d39428ded46f4e4022ea7d1942", size = 153734, upload-time = "2025-08-09T07:57:05.549Z" }, + { url = "https://files.pythonhosted.org/packages/50/10/c117806094d2c956ba88958dab680574019abc0c02bcf57b32287afca544/charset_normalizer-3.4.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:a2d08ac246bb48479170408d6c19f6385fa743e7157d716e144cad849b2dd94b", size = 148551, upload-time = "2025-08-09T07:57:06.823Z" }, + { url = "https://files.pythonhosted.org/packages/61/c5/dc3ba772489c453621ffc27e8978a98fe7e41a93e787e5e5bde797f1dddb/charset_normalizer-3.4.3-cp38-cp38-win32.whl", hash = "sha256:ec557499516fc90fd374bf2e32349a2887a876fbf162c160e3c01b6849eaf557", size = 98459, upload-time = "2025-08-09T07:57:08.031Z" }, + { url = "https://files.pythonhosted.org/packages/05/35/bb59b1cd012d7196fc81c2f5879113971efc226a63812c9cf7f89fe97c40/charset_normalizer-3.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:5d8d01eac18c423815ed4f4a2ec3b439d654e55ee4ad610e153cf02faf67ea40", size = 105887, upload-time = "2025-08-09T07:57:09.401Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ca/9a0983dd5c8e9733565cf3db4df2b0a2e9a82659fd8aa2a868ac6e4a991f/charset_normalizer-3.4.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:70bfc5f2c318afece2f5838ea5e4c3febada0be750fcf4775641052bbba14d05", size = 207520, upload-time = "2025-08-09T07:57:11.026Z" }, + { url = "https://files.pythonhosted.org/packages/39/c6/99271dc37243a4f925b09090493fb96c9333d7992c6187f5cfe5312008d2/charset_normalizer-3.4.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23b6b24d74478dc833444cbd927c338349d6ae852ba53a0d02a2de1fce45b96e", size = 147307, upload-time = "2025-08-09T07:57:12.4Z" }, + { url = "https://files.pythonhosted.org/packages/e4/69/132eab043356bba06eb333cc2cc60c6340857d0a2e4ca6dc2b51312886b3/charset_normalizer-3.4.3-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:34a7f768e3f985abdb42841e20e17b330ad3aaf4bb7e7aeeb73db2e70f077b99", size = 160448, upload-time = "2025-08-09T07:57:13.712Z" }, + { url = "https://files.pythonhosted.org/packages/04/9a/914d294daa4809c57667b77470533e65def9c0be1ef8b4c1183a99170e9d/charset_normalizer-3.4.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb731e5deb0c7ef82d698b0f4c5bb724633ee2a489401594c5c88b02e6cb15f7", size = 157758, upload-time = "2025-08-09T07:57:14.979Z" }, + { url = "https://files.pythonhosted.org/packages/b0/a8/6f5bcf1bcf63cb45625f7c5cadca026121ff8a6c8a3256d8d8cd59302663/charset_normalizer-3.4.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:257f26fed7d7ff59921b78244f3cd93ed2af1800ff048c33f624c87475819dd7", size = 152487, upload-time = "2025-08-09T07:57:16.332Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/d3d0e9592f4e504f9dea08b8db270821c909558c353dc3b457ed2509f2fb/charset_normalizer-3.4.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1ef99f0456d3d46a50945c98de1774da86f8e992ab5c77865ea8b8195341fc19", size = 150054, upload-time = "2025-08-09T07:57:17.576Z" }, + { url = "https://files.pythonhosted.org/packages/20/30/5f64fe3981677fe63fa987b80e6c01042eb5ff653ff7cec1b7bd9268e54e/charset_normalizer-3.4.3-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:2c322db9c8c89009a990ef07c3bcc9f011a3269bc06782f916cd3d9eed7c9312", size = 161703, upload-time = "2025-08-09T07:57:20.012Z" }, + { url = "https://files.pythonhosted.org/packages/e1/ef/dd08b2cac9284fd59e70f7d97382c33a3d0a926e45b15fc21b3308324ffd/charset_normalizer-3.4.3-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:511729f456829ef86ac41ca78c63a5cb55240ed23b4b737faca0eb1abb1c41bc", size = 159096, upload-time = "2025-08-09T07:57:21.329Z" }, + { url = "https://files.pythonhosted.org/packages/45/8c/dcef87cfc2b3f002a6478f38906f9040302c68aebe21468090e39cde1445/charset_normalizer-3.4.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:88ab34806dea0671532d3f82d82b85e8fc23d7b2dd12fa837978dad9bb392a34", size = 153852, upload-time = "2025-08-09T07:57:22.608Z" }, + { url = "https://files.pythonhosted.org/packages/63/86/9cbd533bd37883d467fcd1bd491b3547a3532d0fbb46de2b99feeebf185e/charset_normalizer-3.4.3-cp39-cp39-win32.whl", hash = "sha256:16a8770207946ac75703458e2c743631c79c59c5890c80011d536248f8eaa432", size = 99840, upload-time = "2025-08-09T07:57:23.883Z" }, + { url = "https://files.pythonhosted.org/packages/ce/d6/7e805c8e5c46ff9729c49950acc4ee0aeb55efb8b3a56687658ad10c3216/charset_normalizer-3.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:d22dbedd33326a4a5190dd4fe9e9e693ef12160c77382d9e87919bce54f3d4ca", size = 107438, upload-time = "2025-08-09T07:57:25.287Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9' and python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, +] + +[[package]] +name = "flake8" +version = "5.0.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.8.1'", +] +dependencies = [ + { name = "mccabe", marker = "python_full_version < '3.8.1'" }, + { name = "pycodestyle", version = "2.9.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8.1'" }, + { name = "pyflakes", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8.1'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/00/9808c62b2d529cefc69ce4e4a1ea42c0f855effa55817b7327ec5b75e60a/flake8-5.0.4.tar.gz", hash = "sha256:6fbe320aad8d6b95cec8b8e47bc933004678dc63095be98528b7bdd2a9f510db", size = 145862, upload-time = "2022-08-03T23:21:27.108Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/a0/b881b63a17a59d9d07f5c0cc91a29182c8e8a9aa2bde5b3b2b16519c02f4/flake8-5.0.4-py2.py3-none-any.whl", hash = "sha256:7a1cf6b73744f5806ab95e526f6f0d8c01c66d7bbe349562d22dfca20610b248", size = 61897, upload-time = "2022-08-03T23:21:25.027Z" }, +] + +[[package]] +name = "flake8" +version = "7.1.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.8.1' and python_full_version < '3.9'", +] +dependencies = [ + { name = "mccabe", marker = "python_full_version >= '3.8.1' and python_full_version < '3.9'" }, + { name = "pycodestyle", version = "2.12.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8.1' and python_full_version < '3.9'" }, + { name = "pyflakes", version = "3.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8.1' and python_full_version < '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/16/3f2a0bb700ad65ac9663262905a025917c020a3f92f014d2ba8964b4602c/flake8-7.1.2.tar.gz", hash = "sha256:c586ffd0b41540951ae41af572e6790dbd49fc12b3aa2541685d253d9bd504bd", size = 48119, upload-time = "2025-02-16T18:45:44.296Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/f8/08d37b2cd89da306e3520bd27f8a85692122b42b56c0c2c3784ff09c022f/flake8-7.1.2-py2.py3-none-any.whl", hash = "sha256:1cbc62e65536f65e6d754dfe6f1bada7f5cf392d6f5db3c2b85892466c3e7c1a", size = 57745, upload-time = "2025-02-16T18:45:42.351Z" }, +] + +[[package]] +name = "flake8" +version = "7.3.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.9'", +] +dependencies = [ + { name = "mccabe", marker = "python_full_version >= '3.9'" }, + { name = "pycodestyle", version = "2.14.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "pyflakes", version = "3.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/af/fbfe3c4b5a657d79e5c47a2827a362f9e1b763336a52f926126aa6dc7123/flake8-7.3.0.tar.gz", hash = "sha256:fe044858146b9fc69b551a4b490d69cf960fcb78ad1edcb84e7fbb1b4a8e3872", size = 48326, upload-time = "2025-06-20T19:31:35.838Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/56/13ab06b4f93ca7cac71078fbe37fcea175d3216f31f85c3168a6bbd0bb9a/flake8-7.3.0-py2.py3-none-any.whl", hash = "sha256:b9696257b9ce8beb888cdbe31cf885c90d31928fe202be0889a7cdafad32f01e", size = 57922, upload-time = "2025-06-20T19:31:34.425Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", version = "4.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "anyio", version = "4.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, +] + +[[package]] +name = "keboola-http-client" +version = "1.1.5" +source = { virtual = "." } +dependencies = [ + { name = "aiolimiter" }, + { name = "httpx" }, + { name = "requests", version = "2.32.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "requests", version = "2.32.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, +] + +[package.dev-dependencies] +dev = [ + { name = "flake8", version = "5.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8.1'" }, + { name = "flake8", version = "7.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8.1' and python_full_version < '3.9'" }, + { name = "flake8", version = "7.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "pytest", version = "8.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "aiolimiter", specifier = ">=1.2.1" }, + { name = "httpx", specifier = ">=0.28.1" }, + { name = "requests", specifier = ">=2.32.4" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "flake8", specifier = ">=5.0.4" }, + { name = "pytest", specifier = ">=8.3.5" }, + { name = "ruff", specifier = ">=0.13.2" }, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.8.1' and python_full_version < '3.9'", + "python_full_version < '3.8.1'", +] +sdist = { url = "https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1", size = 67955, upload-time = "2024-04-20T21:34:42.531Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556, upload-time = "2024-04-20T21:34:40.434Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pycodestyle" +version = "2.9.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.8.1'", +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/83/5bcaedba1f47200f0665ceb07bcb00e2be123192742ee0edfb66b600e5fd/pycodestyle-2.9.1.tar.gz", hash = "sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785", size = 102127, upload-time = "2022-08-03T23:13:29.715Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/e4/fc77f1039c34b3612c4867b69cbb2b8a4e569720b1f19b0637002ee03aff/pycodestyle-2.9.1-py2.py3-none-any.whl", hash = "sha256:d1735fc58b418fd7c5f658d28d943854f8a849b01a5d0a1e6f3f3fdd0166804b", size = 41493, upload-time = "2022-08-03T23:13:27.416Z" }, +] + +[[package]] +name = "pycodestyle" +version = "2.12.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.8.1' and python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/43/aa/210b2c9aedd8c1cbeea31a50e42050ad56187754b34eb214c46709445801/pycodestyle-2.12.1.tar.gz", hash = "sha256:6838eae08bbce4f6accd5d5572075c63626a15ee3e6f842df996bf62f6d73521", size = 39232, upload-time = "2024-08-04T20:26:54.576Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/d8/a211b3f85e99a0daa2ddec96c949cac6824bd305b040571b82a03dd62636/pycodestyle-2.12.1-py2.py3-none-any.whl", hash = "sha256:46f0fb92069a7c28ab7bb558f05bfc0110dac69a0cd23c61ea0040283a9d78b3", size = 31284, upload-time = "2024-08-04T20:26:53.173Z" }, +] + +[[package]] +name = "pycodestyle" +version = "2.14.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/11/e0/abfd2a0d2efe47670df87f3e3a0e2edda42f055053c85361f19c0e2c1ca8/pycodestyle-2.14.0.tar.gz", hash = "sha256:c4b5b517d278089ff9d0abdec919cd97262a3367449ea1c8b49b91529167b783", size = 39472, upload-time = "2025-06-20T18:49:48.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/27/a58ddaf8c588a3ef080db9d0b7e0b97215cee3a45df74f3a94dbbf5c893a/pycodestyle-2.14.0-py2.py3-none-any.whl", hash = "sha256:dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d", size = 31594, upload-time = "2025-06-20T18:49:47.491Z" }, +] + +[[package]] +name = "pyflakes" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.8.1'", +] +sdist = { url = "https://files.pythonhosted.org/packages/07/92/f0cb5381f752e89a598dd2850941e7f570ac3cb8ea4a344854de486db152/pyflakes-2.5.0.tar.gz", hash = "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3", size = 66388, upload-time = "2022-07-30T17:29:05.816Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/13/63178f59f74e53acc2165aee4b002619a3cfa7eeaeac989a9eb41edf364e/pyflakes-2.5.0-py2.py3-none-any.whl", hash = "sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2", size = 66116, upload-time = "2022-07-30T17:29:04.179Z" }, +] + +[[package]] +name = "pyflakes" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.8.1' and python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/57/f9/669d8c9c86613c9d568757c7f5824bd3197d7b1c6c27553bc5618a27cce2/pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f", size = 63788, upload-time = "2024-01-05T00:28:47.703Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/d7/f1b7db88d8e4417c5d47adad627a93547f44bdc9028372dbd2313f34a855/pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a", size = 62725, upload-time = "2024-01-05T00:28:45.903Z" }, +] + +[[package]] +name = "pyflakes" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/45/dc/fd034dc20b4b264b3d015808458391acbf9df40b1e54750ef175d39180b1/pyflakes-3.4.0.tar.gz", hash = "sha256:b24f96fafb7d2ab0ec5075b7350b3d2d2218eab42003821c06344973d3ea2f58", size = 64669, upload-time = "2025-06-20T18:45:27.834Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/2f/81d580a0fb83baeb066698975cb14a618bdbed7720678566f1b046a95fe8/pyflakes-3.4.0-py2.py3-none-any.whl", hash = "sha256:f742a7dbd0d9cb9ea41e9a24a918996e8170c799fa528688d40dd582c8265f4f", size = 63551, upload-time = "2025-06-20T18:45:26.937Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "8.3.5" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.8.1' and python_full_version < '3.9'", + "python_full_version < '3.8.1'", +] +dependencies = [ + { name = "colorama", marker = "python_full_version < '3.9' and sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.9'" }, + { name = "iniconfig", marker = "python_full_version < '3.9'" }, + { name = "packaging", marker = "python_full_version < '3.9'" }, + { name = "pluggy", version = "1.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "tomli", marker = "python_full_version < '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/3c/c9d525a414d506893f0cd8a8d0de7706446213181570cdbd766691164e40/pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845", size = 1450891, upload-time = "2025-03-02T12:54:54.503Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/3d/64ad57c803f1fa1e963a7946b6e0fea4a70df53c1a7fed304586539c2bac/pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820", size = 343634, upload-time = "2025-03-02T12:54:52.069Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.9'", +] +dependencies = [ + { name = "colorama", marker = "python_full_version >= '3.9' and sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version >= '3.9' and python_full_version < '3.11'" }, + { name = "iniconfig", marker = "python_full_version >= '3.9'" }, + { name = "packaging", marker = "python_full_version >= '3.9'" }, + { name = "pluggy", version = "1.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "pygments", marker = "python_full_version >= '3.9'" }, + { name = "tomli", marker = "python_full_version >= '3.9' and python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "requests" +version = "2.32.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.8.1' and python_full_version < '3.9'", + "python_full_version < '3.8.1'", +] +dependencies = [ + { name = "certifi", marker = "python_full_version < '3.9'" }, + { name = "charset-normalizer", marker = "python_full_version < '3.9'" }, + { name = "idna", marker = "python_full_version < '3.9'" }, + { name = "urllib3", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/0a/929373653770d8a0d7ea76c37de6e41f11eb07559b103b1c02cafb3f7cf8/requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422", size = 135258, upload-time = "2025-06-09T16:43:07.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", size = 64847, upload-time = "2025-06-09T16:43:05.728Z" }, +] + +[[package]] +name = "requests" +version = "2.32.5" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.9'", +] +dependencies = [ + { name = "certifi", marker = "python_full_version >= '3.9'" }, + { name = "charset-normalizer", marker = "python_full_version >= '3.9'" }, + { name = "idna", marker = "python_full_version >= '3.9'" }, + { name = "urllib3", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, +] + +[[package]] +name = "ruff" +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/df/8d7d8c515d33adfc540e2edf6c6021ea1c5a58a678d8cfce9fae59aabcab/ruff-0.13.2.tar.gz", hash = "sha256:cb12fffd32fb16d32cef4ed16d8c7cdc27ed7c944eaa98d99d01ab7ab0b710ff", size = 5416417, upload-time = "2025-09-25T14:54:09.936Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/84/5716a7fa4758e41bf70e603e13637c42cfb9dbf7ceb07180211b9bbf75ef/ruff-0.13.2-py3-none-linux_armv6l.whl", hash = "sha256:3796345842b55f033a78285e4f1641078f902020d8450cade03aad01bffd81c3", size = 12343254, upload-time = "2025-09-25T14:53:27.784Z" }, + { url = "https://files.pythonhosted.org/packages/9b/77/c7042582401bb9ac8eff25360e9335e901d7a1c0749a2b28ba4ecb239991/ruff-0.13.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ff7e4dda12e683e9709ac89e2dd436abf31a4d8a8fc3d89656231ed808e231d2", size = 13040891, upload-time = "2025-09-25T14:53:31.38Z" }, + { url = "https://files.pythonhosted.org/packages/c6/15/125a7f76eb295cb34d19c6778e3a82ace33730ad4e6f28d3427e134a02e0/ruff-0.13.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c75e9d2a2fafd1fdd895d0e7e24b44355984affdde1c412a6f6d3f6e16b22d46", size = 12243588, upload-time = "2025-09-25T14:53:33.543Z" }, + { url = "https://files.pythonhosted.org/packages/9e/eb/0093ae04a70f81f8be7fd7ed6456e926b65d238fc122311293d033fdf91e/ruff-0.13.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cceac74e7bbc53ed7d15d1042ffe7b6577bf294611ad90393bf9b2a0f0ec7cb6", size = 12491359, upload-time = "2025-09-25T14:53:35.892Z" }, + { url = "https://files.pythonhosted.org/packages/43/fe/72b525948a6956f07dad4a6f122336b6a05f2e3fd27471cea612349fedb9/ruff-0.13.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6ae3f469b5465ba6d9721383ae9d49310c19b452a161b57507764d7ef15f4b07", size = 12162486, upload-time = "2025-09-25T14:53:38.171Z" }, + { url = "https://files.pythonhosted.org/packages/6a/e3/0fac422bbbfb2ea838023e0d9fcf1f30183d83ab2482800e2cb892d02dfe/ruff-0.13.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f8f9e3cd6714358238cd6626b9d43026ed19c0c018376ac1ef3c3a04ffb42d8", size = 13871203, upload-time = "2025-09-25T14:53:41.943Z" }, + { url = "https://files.pythonhosted.org/packages/6b/82/b721c8e3ec5df6d83ba0e45dcf00892c4f98b325256c42c38ef136496cbf/ruff-0.13.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:c6ed79584a8f6cbe2e5d7dbacf7cc1ee29cbdb5df1172e77fbdadc8bb85a1f89", size = 14929635, upload-time = "2025-09-25T14:53:43.953Z" }, + { url = "https://files.pythonhosted.org/packages/c4/a0/ad56faf6daa507b83079a1ad7a11694b87d61e6bf01c66bd82b466f21821/ruff-0.13.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aed130b2fde049cea2019f55deb939103123cdd191105f97a0599a3e753d61b0", size = 14338783, upload-time = "2025-09-25T14:53:46.205Z" }, + { url = "https://files.pythonhosted.org/packages/47/77/ad1d9156db8f99cd01ee7e29d74b34050e8075a8438e589121fcd25c4b08/ruff-0.13.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1887c230c2c9d65ed1b4e4cfe4d255577ea28b718ae226c348ae68df958191aa", size = 13355322, upload-time = "2025-09-25T14:53:48.164Z" }, + { url = "https://files.pythonhosted.org/packages/64/8b/e87cfca2be6f8b9f41f0bb12dc48c6455e2d66df46fe61bb441a226f1089/ruff-0.13.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5bcb10276b69b3cfea3a102ca119ffe5c6ba3901e20e60cf9efb53fa417633c3", size = 13354427, upload-time = "2025-09-25T14:53:50.486Z" }, + { url = "https://files.pythonhosted.org/packages/7f/df/bf382f3fbead082a575edb860897287f42b1b3c694bafa16bc9904c11ed3/ruff-0.13.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:afa721017aa55a555b2ff7944816587f1cb813c2c0a882d158f59b832da1660d", size = 13537637, upload-time = "2025-09-25T14:53:52.887Z" }, + { url = "https://files.pythonhosted.org/packages/51/70/1fb7a7c8a6fc8bd15636288a46e209e81913b87988f26e1913d0851e54f4/ruff-0.13.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1dbc875cf3720c64b3990fef8939334e74cb0ca65b8dbc61d1f439201a38101b", size = 12340025, upload-time = "2025-09-25T14:53:54.88Z" }, + { url = "https://files.pythonhosted.org/packages/4c/27/1e5b3f1c23ca5dd4106d9d580e5c13d9acb70288bff614b3d7b638378cc9/ruff-0.13.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:5b939a1b2a960e9742e9a347e5bbc9b3c3d2c716f86c6ae273d9cbd64f193f22", size = 12133449, upload-time = "2025-09-25T14:53:57.089Z" }, + { url = "https://files.pythonhosted.org/packages/2d/09/b92a5ccee289f11ab128df57d5911224197d8d55ef3bd2043534ff72ca54/ruff-0.13.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:50e2d52acb8de3804fc5f6e2fa3ae9bdc6812410a9e46837e673ad1f90a18736", size = 13051369, upload-time = "2025-09-25T14:53:59.124Z" }, + { url = "https://files.pythonhosted.org/packages/89/99/26c9d1c7d8150f45e346dc045cc49f23e961efceb4a70c47dea0960dea9a/ruff-0.13.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3196bc13ab2110c176b9a4ae5ff7ab676faaa1964b330a1383ba20e1e19645f2", size = 13523644, upload-time = "2025-09-25T14:54:01.622Z" }, + { url = "https://files.pythonhosted.org/packages/f7/00/e7f1501e81e8ec290e79527827af1d88f541d8d26151751b46108978dade/ruff-0.13.2-py3-none-win32.whl", hash = "sha256:7c2a0b7c1e87795fec3404a485096bcd790216c7c146a922d121d8b9c8f1aaac", size = 12245990, upload-time = "2025-09-25T14:54:03.647Z" }, + { url = "https://files.pythonhosted.org/packages/ee/bd/d9f33a73de84fafd0146c6fba4f497c4565fe8fa8b46874b8e438869abc2/ruff-0.13.2-py3-none-win_amd64.whl", hash = "sha256:17d95fb32218357c89355f6f6f9a804133e404fc1f65694372e02a557edf8585", size = 13324004, upload-time = "2025-09-25T14:54:06.05Z" }, + { url = "https://files.pythonhosted.org/packages/c3/12/28fa2f597a605884deb0f65c1b1ae05111051b2a7030f5d8a4ff7f4599ba/ruff-0.13.2-py3-none-win_arm64.whl", hash = "sha256:da711b14c530412c827219312b7d7fbb4877fb31150083add7e8c5336549cea7", size = 12484437, upload-time = "2025-09-25T14:54:08.022Z" }, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + +[[package]] +name = "tomli" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175, upload-time = "2024-11-27T22:38:36.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077, upload-time = "2024-11-27T22:37:54.956Z" }, + { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429, upload-time = "2024-11-27T22:37:56.698Z" }, + { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067, upload-time = "2024-11-27T22:37:57.63Z" }, + { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030, upload-time = "2024-11-27T22:37:59.344Z" }, + { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898, upload-time = "2024-11-27T22:38:00.429Z" }, + { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894, upload-time = "2024-11-27T22:38:02.094Z" }, + { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319, upload-time = "2024-11-27T22:38:03.206Z" }, + { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273, upload-time = "2024-11-27T22:38:04.217Z" }, + { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310, upload-time = "2024-11-27T22:38:05.908Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309, upload-time = "2024-11-27T22:38:06.812Z" }, + { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762, upload-time = "2024-11-27T22:38:07.731Z" }, + { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453, upload-time = "2024-11-27T22:38:09.384Z" }, + { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486, upload-time = "2024-11-27T22:38:10.329Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349, upload-time = "2024-11-27T22:38:11.443Z" }, + { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159, upload-time = "2024-11-27T22:38:13.099Z" }, + { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243, upload-time = "2024-11-27T22:38:14.766Z" }, + { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645, upload-time = "2024-11-27T22:38:15.843Z" }, + { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584, upload-time = "2024-11-27T22:38:17.645Z" }, + { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875, upload-time = "2024-11-27T22:38:19.159Z" }, + { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418, upload-time = "2024-11-27T22:38:20.064Z" }, + { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708, upload-time = "2024-11-27T22:38:21.659Z" }, + { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582, upload-time = "2024-11-27T22:38:22.693Z" }, + { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543, upload-time = "2024-11-27T22:38:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691, upload-time = "2024-11-27T22:38:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170, upload-time = "2024-11-27T22:38:27.921Z" }, + { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530, upload-time = "2024-11-27T22:38:29.591Z" }, + { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666, upload-time = "2024-11-27T22:38:30.639Z" }, + { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954, upload-time = "2024-11-27T22:38:31.702Z" }, + { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724, upload-time = "2024-11-27T22:38:32.837Z" }, + { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383, upload-time = "2024-11-27T22:38:34.455Z" }, + { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.13.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.8.1' and python_full_version < '3.9'", + "python_full_version < '3.8.1'", +] +sdist = { url = "https://files.pythonhosted.org/packages/f6/37/23083fcd6e35492953e8d2aaaa68b860eb422b34627b13f2ce3eb6106061/typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef", size = 106967, upload-time = "2025-04-10T14:19:05.416Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/54/b1ae86c0973cc6f0210b53d508ca3641fb6d0c56823f288d108bc7ab3cc8/typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c", size = 45806, upload-time = "2025-04-10T14:19:03.967Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "urllib3" +version = "2.2.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.8.1' and python_full_version < '3.9'", + "python_full_version < '3.8.1'", +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/63/22ba4ebfe7430b76388e7cd448d5478814d3032121827c12a2cc287e2260/urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9", size = 300677, upload-time = "2024-09-12T10:52:18.401Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/d9/5f4c13cecde62396b0d3fe530a50ccea91e7dfc1ccf0e09c228841bb5ba8/urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac", size = 126338, upload-time = "2024-09-12T10:52:16.589Z" }, +] + +[[package]] +name = "urllib3" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, +] From e15824ca278fdca9cfbd758fac1a5a5643df7f95 Mon Sep 17 00:00:00 2001 From: soustruh Date: Fri, 3 Oct 2025 00:33:52 +0200 Subject: [PATCH 54/56] from __future__ import annotations for legacy Python versions --- src/keboola/http_client/async_client.py | 2 ++ src/keboola/http_client/http.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/keboola/http_client/async_client.py b/src/keboola/http_client/async_client.py index 134ce50..022e1b9 100644 --- a/src/keboola/http_client/async_client.py +++ b/src/keboola/http_client/async_client.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import logging from typing import Any diff --git a/src/keboola/http_client/http.py b/src/keboola/http_client/http.py index ed2d231..79c715c 100644 --- a/src/keboola/http_client/http.py +++ b/src/keboola/http_client/http.py @@ -1,13 +1,15 @@ +from __future__ import annotations + import functools import logging import urllib.parse as urlparse -from http.cookiejar import CookieJar +from http.cookiejar import CookieJar # noqa: F401 - false positive caused by stringified type annotation import requests from requests.adapters import HTTPAdapter from urllib3.util import Retry -Cookie = dict[str, str] | CookieJar +Cookie = "dict[str, str] | CookieJar" METHOD_RETRY_WHITELIST = ("GET", "POST", "PATCH", "UPDATE", "PUT", "DELETE") ALLOWED_METHODS = ["GET", "POST", "PATCH", "UPDATE", "PUT", "DELETE"] From ca4c9b7aa35047f77c2e2fafcc300282615d99b2 Mon Sep 17 00:00:00 2001 From: soustruh Date: Fri, 3 Oct 2025 00:50:53 +0200 Subject: [PATCH 55/56] use the latest actions versions --- .github/workflows/deploy.yml | 4 ++-- .github/workflows/deploy_to_test.yml | 4 ++-- .github/workflows/push_dev.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a1d90fe..a29680c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout 🛒 - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Install uv 💜 uses: astral-sh/setup-uv@v6 @@ -20,7 +20,7 @@ jobs: uses: astral-sh/ruff-action@v3 - name: Set up Python 🐍 - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version: "3.13" diff --git a/.github/workflows/deploy_to_test.yml b/.github/workflows/deploy_to_test.yml index c4e3d95..2430835 100644 --- a/.github/workflows/deploy_to_test.yml +++ b/.github/workflows/deploy_to_test.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout 🛒 - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Install uv 💜 uses: astral-sh/setup-uv@v6 @@ -20,7 +20,7 @@ jobs: uses: astral-sh/ruff-action@v3 - name: Set up Python 🐍 - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version: "3.13" diff --git a/.github/workflows/push_dev.yml b/.github/workflows/push_dev.yml index e2dd037..ee79025 100644 --- a/.github/workflows/push_dev.yml +++ b/.github/workflows/push_dev.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Checkout 🛒 - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Install uv 💜 uses: astral-sh/setup-uv@v6 @@ -21,7 +21,7 @@ jobs: uses: astral-sh/ruff-action@v3 - name: Set up Python ${{ matrix.python-version }} 🐍 - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} From f9db5cf099ba2fab2c66fe83c214af3d60ead3d4 Mon Sep 17 00:00:00 2001 From: soustruh Date: Fri, 3 Oct 2025 14:30:48 +0200 Subject: [PATCH 56/56] version replacement in CI --- .github/workflows/deploy.yml | 9 ++++++++- .github/workflows/deploy_to_test.yml | 9 ++++++++- .github/workflows/push_dev.yml | 8 +++++++- pyproject.toml | 2 +- uv.lock | 2 +- 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a29680c..a250624 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -36,9 +36,16 @@ jobs: run: | uv run pytest tests + - name: Version replacement based on tag ↔️ + if: github.ref_type == 'tag' + run: | + TAG_VERSION=${GITHUB_REF#refs/tags/} + echo "Tag version: $TAG_VERSION" + uv version $TAG_VERSION + - name: Build and publish 🚀 env: - UV_PUBLISH_TOKEN: ${{ secrets.PYPI_PASSWORD }} + UV_PUBLISH_TOKEN: ${{ secrets.UV_PUBLISH_TOKEN_PYPI }} run: | uv build uv publish diff --git a/.github/workflows/deploy_to_test.yml b/.github/workflows/deploy_to_test.yml index 2430835..39dfe13 100644 --- a/.github/workflows/deploy_to_test.yml +++ b/.github/workflows/deploy_to_test.yml @@ -36,9 +36,16 @@ jobs: run: | uv run pytest tests + - name: Version replacement based on tag ↔️ + if: github.ref_type == 'tag' + run: | + TAG_VERSION=${GITHUB_REF#refs/tags/} + echo "Tag version: $TAG_VERSION" + uv version $TAG_VERSION + - name: Build and publish 🚀 env: - UV_PUBLISH_TOKEN: ${{ secrets.PYPI_PASSWORD_TEST }} + UV_PUBLISH_TOKEN: ${{ secrets.UV_PUBLISH_TOKEN_TEST_PYPI }} run: | uv build uv publish --index testpypi diff --git a/.github/workflows/push_dev.yml b/.github/workflows/push_dev.yml index ee79025..2e9af70 100644 --- a/.github/workflows/push_dev.yml +++ b/.github/workflows/push_dev.yml @@ -4,7 +4,6 @@ on: [ push ] jobs: build: - runs-on: ubuntu-latest strategy: matrix: @@ -36,3 +35,10 @@ jobs: - name: Test with pytest ✅ run: | uv run pytest tests + + - name: Version replacement based on tag ↔️ + if: github.ref_type == 'tag' + run: | + TAG_VERSION=${GITHUB_REF#refs/tags/} + echo "Tag version: $TAG_VERSION" + uv version $TAG_VERSION diff --git a/pyproject.toml b/pyproject.toml index 6e4b095..93b320e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "keboola.http-client" -version = "1.1.5" +version = "0.0.0" # replaced by the actual version based on the release tag in github actions dependencies = [ "aiolimiter>=1.2.1", "httpx>=0.28.1", diff --git a/uv.lock b/uv.lock index c67ec64..88ced01 100644 --- a/uv.lock +++ b/uv.lock @@ -279,7 +279,7 @@ wheels = [ [[package]] name = "keboola-http-client" -version = "1.1.5" +version = "0.0.0" source = { virtual = "." } dependencies = [ { name = "aiolimiter" },