From 6adce017d9579cf559c983037be0120091be42e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=9B=E6=B0=B4=E5=B1=85=E5=AE=A4?= Date: Wed, 6 Sep 2023 10:45:15 +0800 Subject: [PATCH 1/5] :sparkles: Use httpx for enkanetwork --- core/services/players/services.py | 5 +- plugins/genshin/player_cards.py | 3 +- utils/enkanetwork.py | 95 ++++++++++++++++++++++++++++++- 3 files changed, 95 insertions(+), 8 deletions(-) diff --git a/core/services/players/services.py b/core/services/players/services.py index cdaed6fc9..7f764afae 100644 --- a/core/services/players/services.py +++ b/core/services/players/services.py @@ -3,7 +3,6 @@ from aiohttp import ClientConnectorError from enkanetwork import ( - EnkaNetworkAPI, VaildateUIDError, HTTPException, EnkaPlayerNotFound, @@ -15,12 +14,10 @@ from core.dependence.redisdb import RedisDB from core.services.players.models import PlayersDataBase as Player, PlayerInfoSQLModel, PlayerInfo from core.services.players.repositories import PlayerInfoRepository -from utils.enkanetwork import RedisCache +from utils.enkanetwork import RedisCache, EnkaNetworkAPI from utils.log import logger from utils.patch.aiohttp import AioHttpTimeoutException -from gram_core.services.players.services import PlayersService - __all__ = ("PlayersService", "PlayerInfoService") diff --git a/plugins/genshin/player_cards.py b/plugins/genshin/player_cards.py index 19ad123c1..131e1091a 100644 --- a/plugins/genshin/player_cards.py +++ b/plugins/genshin/player_cards.py @@ -3,7 +3,6 @@ from enkanetwork import ( DigitType, - EnkaNetworkAPI, EnkaNetworkResponse, EnkaServerError, Equipments, @@ -33,7 +32,7 @@ from metadata.shortname import roleToName from modules.playercards.file import PlayerCardsFile from modules.playercards.helpers import ArtifactStatsTheory -from utils.enkanetwork import RedisCache +from utils.enkanetwork import RedisCache, EnkaNetworkAPI from utils.helpers import download_resource from utils.log import logger from utils.patch.aiohttp import AioHttpTimeoutException diff --git a/utils/enkanetwork.py b/utils/enkanetwork.py index 50c47aa74..f75f909fb 100644 --- a/utils/enkanetwork.py +++ b/utils/enkanetwork.py @@ -1,13 +1,18 @@ +import logging +import warnings from typing import Dict, Any, Optional, TYPE_CHECKING -from enkanetwork import Cache +from cachetools import TTLCache +from enkanetwork import Cache, EnkaServerError, TimedOut, ERROR_ENKA, EnkaNetworkAPI as _EnkaNetworkAPI, Assets +from enkanetwork.config import Config +from enkanetwork.http import HTTPClient as _HTTPClient, Route +from httpx import AsyncClient, TimeoutException, HTTPError, Timeout try: import ujson as jsonlib except ImportError: import json as jsonlib - if TYPE_CHECKING: from redis import asyncio as aioredis @@ -43,3 +48,89 @@ async def exists(self, key) -> int: async def ttl(self, key) -> int: qname = self.get_qname(key) return await self.redis.ttl(qname) + + +class HTTPClient(_HTTPClient): + async def close(self) -> None: + await self.client.aclose() + + def __init__( + self, *, key: Optional[str] = None, agent: Optional[str] = None, timeout: Optional[Any] = None + ) -> None: + if timeout is None: + timeout = Timeout( + connect=5.0, + read=5.0, + write=5.0, + pool=1.0, + ) + + if agent is not None: + Config.init_user_agent(agent) + agent = agent or Config.USER_AGENT + if key is None: + warnings.warn("'key' has depercated.") + self.client = AsyncClient(timeout=timeout, headers={"User-Agent": agent}) + + async def request(self, route: Route, **kwargs: Any) -> Any: + method = route.method + url = route.url + username = route.username + + try: + response = await self.client.request(method, url, **kwargs) + except HTTPError as e: + raise EnkaServerError from e + except TimeoutException as e: + raise TimedOut from e + + _host = response.url.host + + if response.is_error: + if _host == Config.ENKA_URL: + err = ERROR_ENKA.get(response.status_code, None) + if err: + raise err[0](err[1].format(uid=username)) + raise EnkaServerError(f"Server error status code: {response.status_code}") + + return {"status": response.status_code, "content": response.content} + + +class StaticCache(Cache): + def __init__(self, maxsize: int, ttl: int) -> None: + self.cache = TTLCache(maxsize, ttl) + + async def get(self, key) -> Dict[str, Any]: + data = self.cache.get(key) + return jsonlib.loads(data) if data is not None else data + + async def set(self, key, value) -> None: + self.cache[key] = jsonlib.dumps(value) + + +class EnkaNetworkAPI(_EnkaNetworkAPI): + def __init__( + self, + *, + lang: str = "en", + debug: bool = False, + key: str = "", + cache: bool = True, + user_agent: str = "", + timeout: int = 10, + ) -> None: # noqa: E501 + # Logging + logging.basicConfig() + logging.getLogger("enkanetwork").setLevel(logging.DEBUG if debug else logging.ERROR) # noqa: E501 + + # Set language and load config + self.assets = Assets(lang) + + # Cache + self._enable_cache = cache + if self._enable_cache: + Config.init_cache(StaticCache(1024, 60 * 1)) + + # http client + self.__http = HTTPClient(key=key, agent=user_agent, timeout=timeout) + self._closed = False From 245df1c87b75f40333f2c3e8b1c48fbde4ead75e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=9B=E6=B0=B4=E5=B1=85=E5=AE=A4?= Date: Wed, 6 Sep 2023 10:53:06 +0800 Subject: [PATCH 2/5] :bug: Fix bug --- core/services/players/services.py | 1 + utils/enkanetwork.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/services/players/services.py b/core/services/players/services.py index 7f764afae..75e6caf7b 100644 --- a/core/services/players/services.py +++ b/core/services/players/services.py @@ -14,6 +14,7 @@ from core.dependence.redisdb import RedisDB from core.services.players.models import PlayersDataBase as Player, PlayerInfoSQLModel, PlayerInfo from core.services.players.repositories import PlayerInfoRepository +from gram_core.services.players.services import PlayersService from utils.enkanetwork import RedisCache, EnkaNetworkAPI from utils.log import logger from utils.patch.aiohttp import AioHttpTimeoutException diff --git a/utils/enkanetwork.py b/utils/enkanetwork.py index f75f909fb..f3c8b768d 100644 --- a/utils/enkanetwork.py +++ b/utils/enkanetwork.py @@ -52,7 +52,8 @@ async def ttl(self, key) -> int: class HTTPClient(_HTTPClient): async def close(self) -> None: - await self.client.aclose() + if not self.client.is_closed: + await self.client.aclose() def __init__( self, *, key: Optional[str] = None, agent: Optional[str] = None, timeout: Optional[Any] = None From 51d29c7e1c44e7cc5ec8460eeca33057e771d3f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=9B=E6=B0=B4=E5=B1=85=E5=AE=A4?= Date: Wed, 6 Sep 2023 10:58:03 +0800 Subject: [PATCH 3/5] :bug: Fix bug --- utils/enkanetwork.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/utils/enkanetwork.py b/utils/enkanetwork.py index f3c8b768d..8ec7c843a 100644 --- a/utils/enkanetwork.py +++ b/utils/enkanetwork.py @@ -3,8 +3,11 @@ from typing import Dict, Any, Optional, TYPE_CHECKING from cachetools import TTLCache -from enkanetwork import Cache, EnkaServerError, TimedOut, ERROR_ENKA, EnkaNetworkAPI as _EnkaNetworkAPI, Assets +from enkanetwork.assets import Assets +from enkanetwork.cache import Cache +from enkanetwork.client import EnkaNetworkAPI as _EnkaNetworkAPI from enkanetwork.config import Config +from enkanetwork.exception import TimedOut, NetworkError, EnkaServerError, ERROR_ENKA from enkanetwork.http import HTTPClient as _HTTPClient, Route from httpx import AsyncClient, TimeoutException, HTTPError, Timeout @@ -80,10 +83,10 @@ async def request(self, route: Route, **kwargs: Any) -> Any: try: response = await self.client.request(method, url, **kwargs) - except HTTPError as e: - raise EnkaServerError from e except TimeoutException as e: raise TimedOut from e + except HTTPError as e: + raise NetworkError from e _host = response.url.host @@ -133,5 +136,5 @@ def __init__( Config.init_cache(StaticCache(1024, 60 * 1)) # http client - self.__http = HTTPClient(key=key, agent=user_agent, timeout=timeout) + self.__http = HTTPClient(key=key, agent=user_agent, timeout=timeout) # skipcq: PTC-W0037 self._closed = False From 4de53a980ffb7fe6dfde7c7f3c4c6eeaa63ce28e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=9B=E6=B0=B4=E5=B1=85=E5=AE=A4?= Date: Wed, 6 Sep 2023 11:00:01 +0800 Subject: [PATCH 4/5] :art: Update import --- core/services/players/services.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/services/players/services.py b/core/services/players/services.py index 75e6caf7b..bce1e1227 100644 --- a/core/services/players/services.py +++ b/core/services/players/services.py @@ -14,11 +14,12 @@ from core.dependence.redisdb import RedisDB from core.services.players.models import PlayersDataBase as Player, PlayerInfoSQLModel, PlayerInfo from core.services.players.repositories import PlayerInfoRepository -from gram_core.services.players.services import PlayersService from utils.enkanetwork import RedisCache, EnkaNetworkAPI from utils.log import logger from utils.patch.aiohttp import AioHttpTimeoutException +from gram_core.services.players.services import PlayersService + __all__ = ("PlayersService", "PlayerInfoService") From 46023be897343ec71cf2530aab855c035e5d9974 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=9B=E6=B0=B4=E5=B1=85=E5=AE=A4?= Date: Wed, 6 Sep 2023 11:04:11 +0800 Subject: [PATCH 5/5] :art: Use `enkanetwork.TimedOut` --- core/services/players/services.py | 4 ++-- plugins/genshin/player_cards.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/services/players/services.py b/core/services/players/services.py index bce1e1227..bb1f636d8 100644 --- a/core/services/players/services.py +++ b/core/services/players/services.py @@ -7,6 +7,7 @@ HTTPException, EnkaPlayerNotFound, PlayerInfo as EnkaPlayerInfo, + TimedOut, ) from core.base_service import BaseService @@ -16,7 +17,6 @@ from core.services.players.repositories import PlayerInfoRepository from utils.enkanetwork import RedisCache, EnkaNetworkAPI from utils.log import logger -from utils.patch.aiohttp import AioHttpTimeoutException from gram_core.services.players.services import PlayersService @@ -49,7 +49,7 @@ async def get_player_info_from_enka(self, player_id: int) -> Optional[EnkaPlayer return response.player except (VaildateUIDError, EnkaPlayerNotFound, HTTPException) as exc: logger.warning("EnkaNetwork 请求失败: %s", str(exc)) - except AioHttpTimeoutException as exc: + except TimedOut as exc: logger.warning("EnkaNetwork 请求超时: %s", str(exc)) except ClientConnectorError as exc: logger.warning("EnkaNetwork 请求错误: %s", str(exc)) diff --git a/plugins/genshin/player_cards.py b/plugins/genshin/player_cards.py index 131e1091a..3d6039e9f 100644 --- a/plugins/genshin/player_cards.py +++ b/plugins/genshin/player_cards.py @@ -15,6 +15,7 @@ EnkaServerUnknown, EnkaServerRateLimit, EnkaPlayerNotFound, + TimedOut, ) from pydantic import BaseModel from telegram import InlineKeyboardButton, InlineKeyboardMarkup @@ -35,7 +36,6 @@ from utils.enkanetwork import RedisCache, EnkaNetworkAPI from utils.helpers import download_resource from utils.log import logger -from utils.patch.aiohttp import AioHttpTimeoutException from utils.uid import mask_number if TYPE_CHECKING: @@ -76,7 +76,7 @@ async def _update_enka_data(self, uid) -> Union[EnkaNetworkResponse, str]: data = await self.player_cards_file.merge_info(uid, data) await self.cache.set(uid, data) return EnkaNetworkResponse.parse_obj(data) - except AioHttpTimeoutException: + except TimedOut: error = "Enka.Network 服务请求超时，请稍后重试" except EnkaServerRateLimit: error = "Enka.Network 已对此API进行速率限制，请稍后重试"