([\s\S]*?)<\/answer>',
]
-
+
for regex in extract_regexes:
match = re.search(regex, line)
if match:
@@ -283,7 +286,7 @@ def __init__(self):
self.available_models = [
"MBZUAI-IFM/K2-Think",
]
-
+
def list(self):
"""Return list of available models"""
return [
@@ -423,6 +426,9 @@ def K2ThinkClient(**kwargs):
stream=True
)
- for chunk in response:
- if chunk.choices[0].delta.content:
- print(chunk.choices[0].delta.content, end='', flush=True)
\ No newline at end of file
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes, ChatCompletion)):
+ for chunk in response:
+ if chunk.choices[0].delta.content:
+ print(chunk.choices[0].delta.content, end='', flush=True)
+ else:
+ print(response)
diff --git a/webscout/Provider/OPENAI/PI.py b/webscout/Provider/OPENAI/PI.py
index 913f0339..b775f93b 100644
--- a/webscout/Provider/OPENAI/PI.py
+++ b/webscout/Provider/OPENAI/PI.py
@@ -1,24 +1,29 @@
-from curl_cffi.requests import Session
-from curl_cffi import CurlError
import json
-import time
-import uuid
import re
import threading
-from typing import List, Dict, Optional, Union, Generator, Any
+import time
+import uuid
+from typing import Any, Dict, Generator, List, Optional, Union
from uuid import uuid4
-# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
-from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage
-)
+from curl_cffi import CurlError
+from curl_cffi.requests import Session
+
+from webscout import exceptions
# Attempt to import LitAgent, fallback if not available
from webscout.litagent import LitAgent
-from webscout import exceptions
+# Import base classes and utility structures
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
+from webscout.Provider.OPENAI.utils import (
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+)
# --- PI.ai Client ---
@@ -51,9 +56,9 @@ def create(
raise ValueError(f"Voice '{voice_name}' not available. Choose from: {list(self._client.AVAILABLE_VOICES.keys())}")
# Use format_prompt from utils.py to convert OpenAI messages format to Pi.ai prompt
- from webscout.Provider.OPENAI.utils import format_prompt, count_tokens
+ from webscout.Provider.OPENAI.utils import count_tokens, format_prompt
prompt = format_prompt(messages, do_continue=True, add_special_tokens=True)
-
+
# Ensure conversation is started
if not self._client.conversation_id:
self._client.start_conversation()
@@ -66,12 +71,12 @@ def create(
if stream:
return self._create_stream(
- request_id, created_time, model, prompt,
+ request_id, created_time, model, prompt,
timeout, proxies, voice, voice_name, output_file, prompt_tokens
)
else:
return self._create_non_stream(
- request_id, created_time, model, prompt,
+ request_id, created_time, model, prompt,
timeout, proxies, voice, voice_name, output_file, prompt_tokens
)
@@ -81,9 +86,9 @@ def _create_stream(
voice: bool = False, voice_name: str = "voice3", output_file: str = "PiAI.mp3",
prompt_tokens: Optional[int] = None
) -> Generator[ChatCompletionChunk, None, None]:
-
+
from webscout.Provider.OPENAI.utils import count_tokens
-
+
data = {
'text': prompt,
'conversation': self._client.conversation_id
@@ -116,7 +121,6 @@ def _create_stream(
# Track token usage across chunks
# prompt_tokens = len(prompt.split()) if prompt else 0
completion_tokens = 0
- total_tokens = prompt_tokens
sids = []
streaming_text = ""
@@ -127,19 +131,18 @@ def _create_stream(
if line_bytes:
line = line_bytes.decode('utf-8')
full_raw_data_for_sids += line + "\n"
-
+
if line.startswith("data: "):
json_line_str = line[6:]
try:
chunk_data = json.loads(json_line_str)
content = chunk_data.get('text', '')
-
+
if content:
# Assume content is incremental (new text since last chunk)
new_content = content
streaming_text += new_content
completion_tokens += count_tokens(new_content) if new_content else 0
- total_tokens = prompt_tokens + completion_tokens
# Create OpenAI-compatible chunk
delta = ChoiceDelta(
@@ -202,9 +205,9 @@ def _create_non_stream(
voice: bool = False, voice_name: str = "voice3", output_file: str = "PiAI.mp3",
prompt_tokens: Optional[int] = None
) -> ChatCompletion:
-
+
from webscout.Provider.OPENAI.utils import count_tokens
-
+
# Collect streaming response into a single response
full_content = ""
# prompt_tokens = len(prompt.split()) if prompt else 0 # replaced
@@ -214,7 +217,7 @@ def _create_non_stream(
prompt_tokens = count_tokens(prompt)
for chunk in self._create_stream(
- request_id, created_time, model, prompt,
+ request_id, created_time, model, prompt,
timeout, proxies, voice, voice_name, output_file, prompt_tokens
):
if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.content:
@@ -260,7 +263,7 @@ def __init__(self, client: 'PiAI'):
class PiAI(OpenAICompatibleProvider):
"""
PiAI provider following OpenAI-compatible interface.
-
+
Supports Pi.ai specific features like voice generation and conversation management.
"""
required_auth = False
@@ -277,7 +280,7 @@ class PiAI(OpenAICompatibleProvider):
}
def __init__(
- self,
+ self,
api_key: Optional[str] = None,
timeout: int = 30,
proxies: Optional[Dict[str, str]] = None,
@@ -285,7 +288,7 @@ def __init__(
):
"""
Initialize PI.ai provider.
-
+
Args:
api_key: Not used for Pi.ai but kept for compatibility
timeout: Request timeout in seconds
@@ -295,11 +298,11 @@ def __init__(
super().__init__(proxies=proxies)
self.timeout = timeout
self.conversation_id = None
-
+
# Setup URLs
self.primary_url = 'https://pi.ai/api/chat'
self.fallback_url = 'https://pi.ai/api/v2/chat'
-
+
# Setup headers
self.headers = {
'Accept': 'text/event-stream',
@@ -315,27 +318,27 @@ def __init__(
'User-Agent': LitAgent().random(),
'X-Api-Version': '3'
}
-
+
# Setup cookies
self.cookies = {
'__cf_bm': uuid4().hex
}
-
+
# Replace the base session with curl_cffi Session
self.session = Session()
-
+
# Configure session
self.session.headers.update(self.headers)
if proxies:
self.session.proxies = proxies
-
+
# Set cookies on the session
for name, value in self.cookies.items():
self.session.cookies.set(name, value)
-
+
# Initialize chat interface
self.chat = Chat(self)
-
+
# Start conversation
self.start_conversation()
@@ -401,7 +404,7 @@ def list(inner_self):
if __name__ == "__main__":
# Test the OpenAI-compatible interface
client = PiAI()
-
+
# Test streaming
print("Testing streaming response:")
response = client.chat.completions.create(
@@ -411,12 +414,15 @@ def list(inner_self):
],
stream=True
)
-
- for chunk in response:
- if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.content:
- print(chunk.choices[0].delta.content, end="", flush=True)
+
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes, ChatCompletion)):
+ for chunk in response:
+ if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.content:
+ print(chunk.choices[0].delta.content, end="", flush=True)
+ else:
+ print(response)
print()
-
+
# Test non-streaming
print("\nTesting non-streaming response:")
response = client.chat.completions.create(
@@ -426,6 +432,9 @@ def list(inner_self):
],
stream=False
)
-
- print(response.choices[0].message.content)
- print(f"Usage: {response.usage}")
\ No newline at end of file
+
+ if isinstance(response, ChatCompletion):
+ print(response.choices[0].message.content)
+ print(f"Usage: {response.usage}")
+ else:
+ print(response)
diff --git a/webscout/Provider/OPENAI/TogetherAI.py b/webscout/Provider/OPENAI/TogetherAI.py
index 0770fa79..2d5d59f0 100644
--- a/webscout/Provider/OPENAI/TogetherAI.py
+++ b/webscout/Provider/OPENAI/TogetherAI.py
@@ -42,7 +42,7 @@ def create(
"""
# Get API key if not already set
if not self._client.headers.get("Authorization"):
- # If no API key is set, we can't proceed.
+ # If no API key is set, we can't proceed.
# The user should have provided it in __init__.
pass
@@ -224,23 +224,23 @@ def get_models(cls, api_key: str = None):
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json"
}
-
+
response = requests.get(
"https://api.together.xyz/v1/models",
headers=headers,
timeout=30
)
-
+
if response.status_code != 200:
return cls.AVAILABLE_MODELS
-
+
data = response.json()
# Together API returns a list of model objects
if isinstance(data, list):
return [model["id"] for model in data if isinstance(model, dict) and "id" in model]
-
+
return cls.AVAILABLE_MODELS
-
+
except Exception:
return cls.AVAILABLE_MODELS
@@ -261,13 +261,13 @@ def __init__(self, api_key: str = None, browser: str = "chrome", proxies: Option
# Initialize LitAgent for consistent fingerprints across requests
self._agent = LitAgent()
self.headers = self._generate_consistent_fingerprint(browser=browser)
-
+
if api_key:
self.headers["Authorization"] = f"Bearer {api_key}"
-
+
self.session.headers.update(self.headers)
self.chat = Chat(self)
-
+
# Try to update models if API key is provided
if api_key:
try:
diff --git a/webscout/Provider/OPENAI/TwoAI.py b/webscout/Provider/OPENAI/TwoAI.py
index f3a8e806..ed09d98d 100644
--- a/webscout/Provider/OPENAI/TwoAI.py
+++ b/webscout/Provider/OPENAI/TwoAI.py
@@ -1,15 +1,17 @@
-from curl_cffi.requests import Session
-from curl_cffi import CurlError
import json
import time
import uuid
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
# Import base classes and utilities from OPENAI provider stack
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
)
@@ -246,4 +248,4 @@ def list(inner_self):
stream=True
)
for chunk in resp:
- print(chunk, end="")
\ No newline at end of file
+ print(chunk, end="")
diff --git a/webscout/Provider/OPENAI/__init__.py b/webscout/Provider/OPENAI/__init__.py
index b72dea98..ff239b16 100644
--- a/webscout/Provider/OPENAI/__init__.py
+++ b/webscout/Provider/OPENAI/__init__.py
@@ -2,77 +2,75 @@
# Static imports for all OPENAI provider modules
# Base classes and utilities
+from webscout.Provider.OPENAI.ai4chat import AI4Chat
+from webscout.Provider.OPENAI.akashgpt import AkashGPT
+from webscout.Provider.OPENAI.algion import Algion
+from webscout.Provider.OPENAI.ayle import Ayle
from webscout.Provider.OPENAI.base import (
- OpenAICompatibleProvider,
BaseChat,
BaseCompletions,
+ FunctionDefinition,
+ FunctionParameters,
+ OpenAICompatibleProvider,
Tool,
ToolDefinition,
- FunctionParameters,
- FunctionDefinition,
)
-
-from webscout.Provider.OPENAI.utils import (
- ChatCompletion,
- ChatCompletionChunk,
- Choice,
- ChoiceDelta,
- ChatCompletionMessage,
- CompletionUsage,
- ToolCall,
- ToolFunction,
- FunctionCall,
- ToolCallType,
- ModelData,
- ModelList,
- format_prompt,
- get_system_prompt,
- get_last_user_message,
- count_tokens,
-)
-
-# Provider implementations
-from webscout.Provider.OPENAI.DeepAI import DeepAI
-from webscout.Provider.OPENAI.hadadxyz import HadadXYZ
-from webscout.Provider.OPENAI.K2Think import K2Think
-from webscout.Provider.OPENAI.PI import PiAI
-from webscout.Provider.OPENAI.TogetherAI import TogetherAI
-from webscout.Provider.OPENAI.TwoAI import TwoAI
-from webscout.Provider.OPENAI.ai4chat import AI4Chat
-from webscout.Provider.OPENAI.akashgpt import AkashGPT
-from webscout.Provider.OPENAI.algion import Algion
from webscout.Provider.OPENAI.cerebras import Cerebras
from webscout.Provider.OPENAI.chatgpt import ChatGPT, ChatGPTReversed
from webscout.Provider.OPENAI.chatsandbox import ChatSandbox
+
+# Provider implementations
+from webscout.Provider.OPENAI.DeepAI import DeepAI
from webscout.Provider.OPENAI.deepinfra import DeepInfra
from webscout.Provider.OPENAI.e2b import E2B
from webscout.Provider.OPENAI.elmo import Elmo
from webscout.Provider.OPENAI.exaai import ExaAI
from webscout.Provider.OPENAI.freeassist import FreeAssist
-from webscout.Provider.OPENAI.ayle import Ayle
-from webscout.Provider.OPENAI.huggingface import HuggingFace
+from webscout.Provider.OPENAI.gradient import Gradient
from webscout.Provider.OPENAI.groq import Groq
+from webscout.Provider.OPENAI.hadadxyz import HadadXYZ
from webscout.Provider.OPENAI.heckai import HeckAI
+from webscout.Provider.OPENAI.huggingface import HuggingFace
from webscout.Provider.OPENAI.ibm import IBM
+from webscout.Provider.OPENAI.K2Think import K2Think
from webscout.Provider.OPENAI.llmchat import LLMChat
from webscout.Provider.OPENAI.llmchatco import LLMChatCo
+from webscout.Provider.OPENAI.meta import Meta
from webscout.Provider.OPENAI.netwrck import Netwrck
from webscout.Provider.OPENAI.nvidia import Nvidia
from webscout.Provider.OPENAI.oivscode import oivscode
+from webscout.Provider.OPENAI.PI import PiAI
+from webscout.Provider.OPENAI.sambanova import Sambanova
from webscout.Provider.OPENAI.sonus import SonusAI
from webscout.Provider.OPENAI.textpollinations import TextPollinations
+from webscout.Provider.OPENAI.TogetherAI import TogetherAI
from webscout.Provider.OPENAI.toolbaz import Toolbaz
+from webscout.Provider.OPENAI.TwoAI import TwoAI
from webscout.Provider.OPENAI.typefully import TypefullyAI
+from webscout.Provider.OPENAI.typliai import TypliAI
+from webscout.Provider.OPENAI.utils import (
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ FunctionCall,
+ ModelData,
+ ModelList,
+ ToolCall,
+ ToolCallType,
+ ToolFunction,
+ count_tokens,
+ format_prompt,
+ get_last_user_message,
+ get_system_prompt,
+)
from webscout.Provider.OPENAI.venice import Venice
from webscout.Provider.OPENAI.wisecat import WiseCat
from webscout.Provider.OPENAI.writecream import Writecream
from webscout.Provider.OPENAI.x0gpt import X0GPT
-from webscout.Provider.OPENAI.yep import YEPCHAT
from webscout.Provider.OPENAI.zenmux import Zenmux
-from webscout.Provider.OPENAI.gradient import Gradient
-from webscout.Provider.OPENAI.sambanova import Sambanova
-from webscout.Provider.OPENAI.meta import Meta
-from webscout.Provider.OPENAI.typliai import TypliAI
# List of all exported names
__all__ = [
diff --git a/webscout/Provider/OPENAI/ai4chat.py b/webscout/Provider/OPENAI/ai4chat.py
index 5a4d9af1..b39a56c8 100644
--- a/webscout/Provider/OPENAI/ai4chat.py
+++ b/webscout/Provider/OPENAI/ai4chat.py
@@ -1,14 +1,20 @@
import time
-import uuid
import urllib.parse
-from curl_cffi.requests import Session, RequestsError
-from typing import List, Dict, Optional, Union, Generator, Any
+import uuid
+from typing import Any, Dict, Generator, List, Optional, Union
+
+from curl_cffi.requests import RequestsError
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ count_tokens,
)
# --- AI4Chat Client ---
@@ -65,7 +71,7 @@ def _create_stream(
full_response = self._get_ai4chat_response(conversation_prompt, country, user_id, timeout=timeout, proxies=proxies)
# Track token usage
- prompt_tokens = count_tokens(conversation_prompt)
+ count_tokens(conversation_prompt)
completion_tokens = 0
# Stream fixed-size character chunks (e.g., 48 chars)
@@ -194,7 +200,7 @@ def _get_ai4chat_response(self, prompt: str, country: str, user_id: str,
original_proxies = self._client.session.proxies
if proxies is not None:
self._client.session.proxies = proxies
-
+
try:
# URL encode parameters
encoded_text = urllib.parse.quote(prompt)
@@ -299,7 +305,7 @@ class _ModelList:
def list(inner_self):
return type(self).AVAILABLE_MODELS
return _ModelList()
-
+
if __name__ == "__main__":
# Example usage
client = AI4Chat()
@@ -310,4 +316,4 @@ def list(inner_self):
{"role": "user", "content": "Hello, how are you?"}
]
)
- print(response.choices[0].message.content)
\ No newline at end of file
+ print(response.choices[0].message.content)
diff --git a/webscout/Provider/OPENAI/akashgpt.py b/webscout/Provider/OPENAI/akashgpt.py
index 061209a0..1f0b5b1f 100644
--- a/webscout/Provider/OPENAI/akashgpt.py
+++ b/webscout/Provider/OPENAI/akashgpt.py
@@ -1,20 +1,24 @@
+import json
+import re
import time
import uuid
-import requests
-import re
-import json
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
+
+# Import LitAgent for user agent generation
+from webscout.litagent import LitAgent
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions, Tool
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ count_tokens,
+ format_prompt,
)
-from webscout.Provider.OPENAI.utils import format_prompt, get_system_prompt
-
-# Import LitAgent for user agent generation
-from webscout.litagent import LitAgent
# AkashGPT constants
AVAILABLE_MODELS = [
@@ -58,7 +62,7 @@ def create(
conversation_prompt = format_prompt(messages, add_special_tokens=True, include_system=True)
# Set up request parameters
- api_key = kwargs.get("api_key", self._client.api_key)
+ kwargs.get("api_key", self._client.api_key)
# Generate request ID and timestamp
request_id = str(uuid.uuid4())
@@ -312,7 +316,7 @@ class AkashGPT(OpenAICompatibleProvider):
print(response.choices[0].message.content)
"""
required_auth = True
-
+
AVAILABLE_MODELS = AVAILABLE_MODELS
def __init__(
@@ -330,7 +334,7 @@ def __init__(
proxies: Optional proxy configuration dict
"""
super().__init__(api_key=api_key, tools=tools, proxies=proxies)
-
+
# Store the api_key for use in completions
self.api_key = api_key
self.timeout = 30
@@ -383,4 +387,4 @@ def list(inner_self):
messages=[{"role": "user", "content": "Hello! How are you?"}]
)
print(response.choices[0].message.content)
- print(f"Usage: {response.usage}")
\ No newline at end of file
+ print(f"Usage: {response.usage}")
diff --git a/webscout/Provider/OPENAI/algion.py b/webscout/Provider/OPENAI/algion.py
index daf96389..a1bd4dc0 100644
--- a/webscout/Provider/OPENAI/algion.py
+++ b/webscout/Provider/OPENAI/algion.py
@@ -1,15 +1,19 @@
import json
import time
import uuid
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
-from curl_cffi.requests import Session
from curl_cffi import CurlError
+from curl_cffi.requests import Session
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
)
try:
@@ -207,7 +211,7 @@ class Algion(OpenAICompatibleProvider):
def get_models(cls, api_key: str = None):
"""Fetch available models from Algion API."""
api_key = api_key or "123123"
-
+
try:
# Use a temporary curl_cffi session for this class method
temp_session = Session()
@@ -215,21 +219,21 @@ def get_models(cls, api_key: str = None):
"Content-Type": "application/json",
"Authorization": f"Bearer {api_key}",
}
-
+
response = temp_session.get(
"https://api.algion.dev/v1/models",
headers=headers,
impersonate="chrome110"
)
-
+
if response.status_code != 200:
raise Exception(f"Failed to fetch models: HTTP {response.status_code}")
-
+
data = response.json()
if "data" in data and isinstance(data["data"], list):
return [model["id"] for model in data["data"]]
raise Exception("Invalid response format from API")
-
+
except (CurlError, Exception) as e:
raise Exception(f"Failed to fetch models: {str(e)}")
@@ -257,7 +261,7 @@ def __init__(self, browser: str = "chrome", api_key: str = "123123"):
self.headers["Authorization"] = f"Bearer {api_key}"
self.session.headers.update(self.headers)
self.chat = Chat(self)
-
+
@property
def models(self):
class _ModelList:
@@ -278,4 +282,4 @@ def list(inner_self):
max_tokens=1000,
stream=False
)
- print(response.choices[0].message.content)
\ No newline at end of file
+ print(response.choices[0].message.content)
diff --git a/webscout/Provider/OPENAI/ayle.py b/webscout/Provider/OPENAI/ayle.py
index e63a63da..6d5a3d83 100644
--- a/webscout/Provider/OPENAI/ayle.py
+++ b/webscout/Provider/OPENAI/ayle.py
@@ -1,20 +1,20 @@
+import json
import time
import uuid
+from typing import Any, Dict, Generator, List, Optional, Union
+
import requests
-import json
-from typing import List, Dict, Optional, Union, Generator, Any
from webscout.litagent import LitAgent
from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
ChatCompletion,
ChatCompletionChunk,
- Choice,
ChatCompletionMessage,
+ Choice,
ChoiceDelta,
CompletionUsage,
- format_prompt,
- count_tokens
+ count_tokens,
)
# ANSI escape codes for formatting
@@ -28,7 +28,7 @@
"endpoint": "https://ayle.chat/api/chat",
"models": [
"gemini-2.5-flash",
- "llama-3.3-70b-versatile",
+ "llama-3.3-70b-versatile",
"llama-3.3-70b",
"tngtech/deepseek-r1t2-chimera:free",
"openai/gpt-oss-120b",
@@ -64,7 +64,7 @@ def create(
"""
# Determine the provider based on the model
provider = self._client._get_provider_from_model(model)
-
+
# Build the appropriate payload
payload = {
"messages": messages,
@@ -101,7 +101,7 @@ def _create_stream(
for line in response.iter_lines():
if not line:
continue
-
+
try:
line_str = line.decode('utf-8')
if line_str.startswith('0:"'):
@@ -109,18 +109,18 @@ def _create_stream(
if content:
streaming_text += content
completion_tokens += count_tokens(content)
-
+
# Create a delta object for this chunk
delta = ChoiceDelta(content=content)
choice = Choice(index=0, delta=delta, finish_reason=None)
-
+
chunk = ChatCompletionChunk(
id=request_id,
choices=[choice],
created=created_time,
model=model,
)
-
+
yield chunk
except (json.JSONDecodeError, UnicodeDecodeError):
continue
@@ -128,14 +128,14 @@ def _create_stream(
# Final chunk with finish_reason
delta = ChoiceDelta(content=None)
choice = Choice(index=0, delta=delta, finish_reason="stop")
-
+
chunk = ChatCompletionChunk(
id=request_id,
choices=[choice],
created=created_time,
model=model,
)
-
+
yield chunk
except requests.exceptions.RequestException as e:
@@ -172,26 +172,26 @@ def _create_non_stream(
prompt_tokens = count_tokens(str(payload.get("messages", "")))
completion_tokens = count_tokens(full_response)
total_tokens = prompt_tokens + completion_tokens
-
+
usage = CompletionUsage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=total_tokens
)
-
+
# Create the message object
message = ChatCompletionMessage(
role="assistant",
content=full_response
)
-
+
# Create the choice object
choice = Choice(
index=0,
message=message,
finish_reason="stop"
)
-
+
# Create the completion object
completion = ChatCompletion(
id=request_id,
@@ -200,7 +200,7 @@ def _create_non_stream(
model=model,
usage=usage,
)
-
+
return completion
except Exception as e:
@@ -226,7 +226,7 @@ class Ayle(OpenAICompatibleProvider):
required_auth = False
AVAILABLE_MODELS = [
"gemini-2.5-flash",
- "llama-3.3-70b-versatile",
+ "llama-3.3-70b-versatile",
"llama-3.3-70b",
"tngtech/deepseek-r1t2-chimera:free",
"openai/gpt-oss-120b",
@@ -253,10 +253,10 @@ def __init__(
self.timeout = timeout
self.temperature = temperature
self.top_p = top_p
-
+
# Initialize LitAgent for user agent generation
agent = LitAgent()
-
+
self.headers = {
"accept": "*/*",
"accept-language": "en-US,en;q=0.9",
@@ -265,11 +265,11 @@ def __init__(
"referer": "https://ayle.chat/",
"user-agent": agent.random(),
}
-
+
self.session = requests.Session()
self.session.headers.update(self.headers)
self.session.cookies.update({"session": uuid.uuid4().hex})
-
+
# Initialize the chat interface
self.chat = Chat(self)
@@ -288,7 +288,7 @@ def _get_provider_from_model(self, model: str) -> str:
for provider, config in MODEL_CONFIGS.items():
if model in config["models"]:
return provider
-
+
# If model not found, use a default model
print(f"{BOLD}Warning: Model '{model}' not found, using default model 'ayle'{RESET}")
return "ayle"
@@ -299,12 +299,12 @@ def convert_model_name(self, model: str) -> str:
"""
if model in self.AVAILABLE_MODELS:
return model
-
+
# Try to find a matching model
for available_model in self.AVAILABLE_MODELS:
if model.lower() in available_model.lower():
return available_model
-
+
# Default to gemini-2.5-flash if no match
print(f"{BOLD}Warning: Model '{model}' not found, using default model 'gemini-2.5-flash'{RESET}")
return "gemini-2.5-flash"
@@ -333,7 +333,7 @@ def convert_model_name(self, model: str) -> str:
],
stream=False
)
-
+
if response and response.choices and response.choices[0].message.content:
status = "✓"
# Truncate response if too long
diff --git a/webscout/Provider/OPENAI/base.py b/webscout/Provider/OPENAI/base.py
index 17e8091f..4ea230fc 100644
--- a/webscout/Provider/OPENAI/base.py
+++ b/webscout/Provider/OPENAI/base.py
@@ -1,13 +1,15 @@
-from abc import ABC, abstractmethod
-from typing import List, Dict, Optional, Union, Generator, Any, TypedDict, Callable
import json
-import requests
+from abc import ABC, abstractmethod
from dataclasses import dataclass
+from typing import Any, Callable, Dict, Generator, List, Optional, TypedDict, Union
+
+import requests
from litprinter import ic
# Import the utils for response structures
from webscout.Provider.OPENAI.utils import ChatCompletion, ChatCompletionChunk
+
# Define tool-related structures
class ToolDefinition(TypedDict):
"""Definition of a tool that can be called by the AI"""
@@ -32,9 +34,9 @@ class Tool:
name: str
description: str
parameters: Dict[str, Dict[str, Any]]
- required_params: List[str] = None
+ required_params: Optional[List[str]] = None
implementation: Optional[Callable] = None
-
+
def to_dict(self) -> ToolDefinition:
"""Convert to OpenAI tool definition format"""
function_def = {
@@ -46,21 +48,22 @@ def to_dict(self) -> ToolDefinition:
"required": self.required_params or list(self.parameters.keys())
}
}
-
+
return {
"type": "function",
"function": function_def
}
-
+
def execute(self, arguments: Dict[str, Any]) -> Any:
"""Execute the tool with the given arguments"""
if not self.implementation:
return f"Tool '{self.name}' does not have an implementation."
-
+
try:
return self.implementation(**arguments)
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Error executing tool '{self.name}': {str(e)}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Error executing tool '{self.name}': {str(e)}")
return f"Error executing tool '{self.name}': {str(e)}"
class BaseCompletions(ABC):
@@ -82,7 +85,7 @@ def create(
) -> Union[ChatCompletion, Generator[ChatCompletionChunk, None, None]]:
"""
Abstract method to create chat completions with tool support.
-
+
Args:
model: The model to use for completion
messages: List of message dictionaries
@@ -93,16 +96,16 @@ def create(
tools: List of tool definitions available for the model to use
tool_choice: Control over which tool the model should use
**kwargs: Additional model-specific parameters
-
+
Returns:
Either a completion object or a generator of completion chunks if streaming
"""
raise NotImplementedError
-
+
def format_tool_calls(self, tools: List[Union[Tool, Dict[str, Any]]]) -> List[Dict[str, Any]]:
"""Convert tools to the format expected by the provider"""
formatted_tools = []
-
+
for tool in tools:
if isinstance(tool, Tool):
formatted_tools.append(tool.to_dict())
@@ -110,30 +113,31 @@ def format_tool_calls(self, tools: List[Union[Tool, Dict[str, Any]]]) -> List[Di
# Assume already formatted correctly
formatted_tools.append(tool)
else:
- ic.configureOutput(prefix='WARNING| '); ic(f"Skipping invalid tool type: {type(tool)}")
-
+ ic.configureOutput(prefix='WARNING| ')
+ ic(f"Skipping invalid tool type: {type(tool)}")
+
return formatted_tools
-
+
def process_tool_calls(self, tool_calls: List[Dict[str, Any]], available_tools: Dict[str, Tool]) -> List[Dict[str, Any]]:
"""
Process tool calls and execute the relevant tools.
-
+
Args:
tool_calls: List of tool calls from the model
available_tools: Dictionary mapping tool names to their implementations
-
+
Returns:
List of results from executing the tools
"""
results = []
-
+
for call in tool_calls:
try:
function_call = call.get("function", {})
tool_name = function_call.get("name")
arguments_str = function_call.get("arguments", "{}")
-
- # Parse arguments
+
+ # Parse arguments
try:
if isinstance(arguments_str, str):
arguments = json.loads(arguments_str)
@@ -145,7 +149,7 @@ def process_tool_calls(self, tool_calls: List[Dict[str, Any]], available_tools:
"result": f"Error: Could not parse arguments JSON: {arguments_str}"
})
continue
-
+
# Execute the tool if available
if tool_name in available_tools:
tool_result = available_tools[tool_name].execute(arguments)
@@ -159,12 +163,13 @@ def process_tool_calls(self, tool_calls: List[Dict[str, Any]], available_tools:
"result": f"Error: Tool '{tool_name}' not found."
})
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Error processing tool call: {str(e)}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Error processing tool call: {str(e)}")
results.append({
"tool_call_id": call.get("id", "unknown"),
"result": f"Error processing tool call: {str(e)}"
})
-
+
return results
@@ -201,34 +206,34 @@ def models(self):
Subclasses must implement this property.
"""
pass
-
+
def register_tools(self, tools: List[Tool]) -> None:
"""
Register tools with the provider.
-
+
Args:
tools: List of Tool objects to register
"""
for tool in tools:
self.available_tools[tool.name] = tool
-
+
def get_tool_by_name(self, name: str) -> Optional[Tool]:
"""Get a tool by name"""
return self.available_tools.get(name)
-
+
def format_tool_response(self, messages: List[Dict[str, Any]], tool_results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
Format tool results as messages to be sent back to the provider.
-
+
Args:
messages: The original messages
tool_results: Results from executing tools
-
+
Returns:
Updated message list with tool results
"""
updated_messages = messages.copy()
-
+
# Find the assistant message with tool calls
for i, msg in enumerate(reversed(updated_messages)):
if msg.get("role") == "assistant" and "tool_calls" in msg:
@@ -241,5 +246,5 @@ def format_tool_response(self, messages: List[Dict[str, Any]], tool_results: Lis
}
updated_messages.append(tool_message)
break
-
- return updated_messages
\ No newline at end of file
+
+ return updated_messages
diff --git a/webscout/Provider/OPENAI/cerebras.py b/webscout/Provider/OPENAI/cerebras.py
index a38b5553..3b7993d0 100644
--- a/webscout/Provider/OPENAI/cerebras.py
+++ b/webscout/Provider/OPENAI/cerebras.py
@@ -1,15 +1,19 @@
import json
import time
import uuid
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
-from curl_cffi.requests import Session
from curl_cffi import CurlError
+from curl_cffi.requests import Session
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
)
try:
@@ -208,7 +212,7 @@ def get_models(cls, api_key: str = None):
"""Fetch available models from Cerebras API."""
if not api_key:
raise Exception("API key required to fetch models")
-
+
try:
# Use a temporary curl_cffi session for this class method
temp_session = Session()
@@ -216,21 +220,21 @@ def get_models(cls, api_key: str = None):
"Content-Type": "application/json",
"Authorization": f"Bearer {api_key}",
}
-
+
response = temp_session.get(
"https://api.cerebras.ai/v1/models",
headers=headers,
impersonate="chrome120"
)
-
+
if response.status_code != 200:
raise Exception(f"Failed to fetch models: HTTP {response.status_code}")
-
+
data = response.json()
if "data" in data and isinstance(data["data"], list):
return [model['id'] for model in data['data']]
raise Exception("Invalid response format from API")
-
+
except (CurlError, Exception) as e:
raise Exception(f"Failed to fetch models: {str(e)}")
@@ -254,7 +258,7 @@ def __init__(self, browser: str = "chrome", api_key: str = None):
}
self.session.headers.update(self.headers)
self.chat = Chat(self)
-
+
@property
def models(self):
class _ModelList:
@@ -272,4 +276,4 @@ def list(inner_self):
max_tokens=1000,
stream=False
)
- print(response.choices[0].message.content)
\ No newline at end of file
+ print(response.choices[0].message.content)
diff --git a/webscout/Provider/OPENAI/chatgpt.py b/webscout/Provider/OPENAI/chatgpt.py
index 5277a26c..d510cc27 100644
--- a/webscout/Provider/OPENAI/chatgpt.py
+++ b/webscout/Provider/OPENAI/chatgpt.py
@@ -1,18 +1,24 @@
+import base64
+import hashlib
+import json
+import random
import time
import uuid
+from datetime import datetime, timezone
+from typing import Any, Dict, Generator, List, Optional, Union
+
import requests
-import json
-import random
-import base64
-import hashlib
-from datetime import datetime, timedelta, timezone
-from typing import List, Dict, Optional, Union, Generator, Any
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions, Tool
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider, Tool
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ count_tokens,
)
# ANSI escape codes for formatting
@@ -129,24 +135,24 @@ def simulate_bypass_headers(self, accept, spoof_address=False, pre_oai_uuid=None
def generate_proof_token(self, seed: str, difficulty: str, user_agent: str = None):
"""
Improved proof-of-work implementation based on gpt4free/g4f/Provider/openai/proofofwork.py
-
+
Args:
seed: The seed string for the challenge
difficulty: The difficulty hex string
user_agent: Optional user agent string
-
+
Returns:
The proof token starting with 'gAAAAAB'
"""
if user_agent is None:
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36"
-
+
screen = random.choice([3008, 4010, 6000]) * random.choice([1, 2, 4])
-
+
# Get current UTC time
now_utc = datetime.now(timezone.utc)
parse_time = now_utc.strftime('%a, %d %b %Y %H:%M:%S GMT')
-
+
proof_token = [
screen, parse_time,
None, 0, user_agent,
@@ -179,14 +185,14 @@ def solve_sentinel_challenge(self, seed, difficulty):
def generate_fake_sentinel_token(self):
"""Generate a fake sentinel token for initial authentication."""
prefix = "gAAAAAC"
-
+
# More realistic screen sizes
screen = random.choice([3008, 4010, 6000]) * random.choice([1, 2, 4])
-
+
# Get current UTC time
now_utc = datetime.now(timezone.utc)
parse_time = now_utc.strftime('%a, %d %b %Y %H:%M:%S GMT')
-
+
config = [
screen,
parse_time,
@@ -219,7 +225,7 @@ def parse_response(self, input_text):
json_data["message"].get("status") == "finished_successfully" and
json_data["message"].get("metadata", {}).get("is_complete")):
return json_data["message"]["content"]["parts"][0]
- except:
+ except Exception:
pass
return input_text # Return raw text if parsing fails or no complete message found
@@ -553,7 +559,6 @@ def _create_streaming(
# Track conversation state
full_content = ""
- finish_reason = None
prompt_tokens = count_tokens(str(messages))
completion_tokens = 0
total_tokens = prompt_tokens
@@ -563,7 +568,7 @@ def _create_streaming(
if line:
if line.startswith("data: "):
data_str = line[6:] # Remove "data: " prefix
-
+
# Handle [DONE] message
if data_str.strip() == "[DONE]":
# Final chunk with finish_reason
@@ -582,26 +587,26 @@ def _create_streaming(
}
yield chunk
break
-
+
try:
data = json.loads(data_str)
-
+
# Handle different types of messages
if data.get("message"):
message = data["message"]
-
+
# Handle assistant responses
if message.get("author", {}).get("role") == "assistant":
content_parts = message.get("content", {}).get("parts", [])
if content_parts:
new_content = content_parts[0]
-
+
# Get the delta (new content since last chunk)
delta_content = new_content[len(full_content):] if new_content.startswith(full_content) else new_content
full_content = new_content
completion_tokens = count_tokens(full_content)
total_tokens = prompt_tokens + completion_tokens
-
+
# Only yield chunk if there's new content
if delta_content:
delta = ChoiceDelta(content=delta_content, role="assistant")
@@ -618,15 +623,15 @@ def _create_streaming(
"total_tokens": total_tokens
}
yield chunk
-
+
# Handle finish status
if message.get("status") == "finished_successfully":
- finish_reason = "stop"
-
+ pass
+
elif data.get("type") == "message_stream_complete":
# Stream is complete
- finish_reason = "stop"
-
+ pass
+
except json.JSONDecodeError:
# Skip invalid JSON lines
continue
@@ -729,20 +734,20 @@ def _create_non_streaming(
if line:
if line.startswith("data: "):
data_str = line[6:] # Remove "data: " prefix
-
+
# Handle [DONE] message
if data_str.strip() == "[DONE]":
break
-
+
try:
data = json.loads(data_str)
-
+
# Handle assistant responses
if data.get("message") and data["message"].get("author", {}).get("role") == "assistant":
content_parts = data["message"].get("content", {}).get("parts", [])
if content_parts:
full_response = content_parts[0]
-
+
except json.JSONDecodeError:
# Skip invalid JSON lines
continue
@@ -843,4 +848,4 @@ def list(inner_self):
messages=[{"role": "user", "content": "How many r in strawberry"}]
)
print(response.choices[0].message.content)
- print()
\ No newline at end of file
+ print()
diff --git a/webscout/Provider/OPENAI/chatsandbox.py b/webscout/Provider/OPENAI/chatsandbox.py
index 42feee8d..fb80428b 100644
--- a/webscout/Provider/OPENAI/chatsandbox.py
+++ b/webscout/Provider/OPENAI/chatsandbox.py
@@ -1,22 +1,24 @@
-from typing import List, Dict, Optional, Union, Generator, Any
-import time
import json
+import time
+from typing import Any, Dict, Generator, List, Optional, Union
+
+from curl_cffi.const import CurlHttpVersion
+from curl_cffi.requests import Session
+
+from webscout import exceptions
+from webscout.AIutel import sanitize_stream
from webscout.litagent import LitAgent
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
ChatCompletion,
ChatCompletionChunk,
- Choice,
ChatCompletionMessage,
+ Choice,
ChoiceDelta,
CompletionUsage,
+ count_tokens,
format_prompt,
- count_tokens
)
-from curl_cffi.requests import Session
-from curl_cffi.const import CurlHttpVersion
-from webscout.AIutel import sanitize_stream
-from webscout import exceptions
# ANSI escape codes for formatting
BOLD = "\033[1m"
@@ -92,7 +94,7 @@ def create(
session = Session()
session.headers.update(headers)
session.proxies = proxies if proxies is not None else {}
-
+
def for_stream():
try:
response = session.post(
@@ -107,7 +109,7 @@ def for_stream():
raise exceptions.FailedToGenerateResponseError(
f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}"
)
-
+
streaming_text = ""
# Use sanitize_stream with the custom extractor
processed_stream = sanitize_stream(
@@ -173,16 +175,16 @@ def __init__(self, client: 'ChatSandbox'):
class ChatSandbox(OpenAICompatibleProvider):
AVAILABLE_MODELS = [
- "openai",
- "openai-gpt-4o",
- "openai-o1-mini",
- "deepseek",
- "deepseek-r1",
+ "openai",
+ "openai-gpt-4o",
+ "openai-o1-mini",
+ "deepseek",
+ "deepseek-r1",
"deepseek-r1-full",
- "gemini",
+ "gemini",
"gemini-thinking",
- "mistral",
- "mistral-large",
+ "mistral",
+ "mistral-large",
"gemma-3",
"llama"
]
diff --git a/webscout/Provider/OPENAI/deepinfra.py b/webscout/Provider/OPENAI/deepinfra.py
index 0147ab66..66089d06 100644
--- a/webscout/Provider/OPENAI/deepinfra.py
+++ b/webscout/Provider/OPENAI/deepinfra.py
@@ -1,14 +1,18 @@
-import requests
import json
import time
import uuid
-import collections
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
+
+import requests
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
)
try:
@@ -16,8 +20,9 @@
except ImportError:
pass
+
class Completions(BaseCompletions):
- def __init__(self, client: 'DeepInfra'):
+ def __init__(self, client: "DeepInfra"):
self._client = client
def create(
@@ -31,7 +36,7 @@ def create(
top_p: Optional[float] = None,
timeout: Optional[int] = None,
proxies: Optional[Dict[str, str]] = None,
- **kwargs: Any
+ **kwargs: Any,
) -> Union[ChatCompletion, Generator[ChatCompletionChunk, None, None]]:
payload = {
"model": model,
@@ -49,11 +54,18 @@ def create(
if stream:
return self._create_stream(request_id, created_time, model, payload, timeout, proxies)
else:
- return self._create_non_stream(request_id, created_time, model, payload, timeout, proxies)
+ return self._create_non_stream(
+ request_id, created_time, model, payload, timeout, proxies
+ )
def _create_stream(
- self, request_id: str, created_time: int, model: str, payload: Dict[str, Any],
- timeout: Optional[int] = None, proxies: Optional[Dict[str, str]] = None
+ self,
+ request_id: str,
+ created_time: int,
+ model: str,
+ payload: Dict[str, Any],
+ timeout: Optional[int] = None,
+ proxies: Optional[Dict[str, str]] = None,
) -> Generator[ChatCompletionChunk, None, None]:
try:
response = self._client.session.post(
@@ -62,7 +74,7 @@ def _create_stream(
json=payload,
stream=True,
timeout=timeout or self._client.timeout,
- proxies=proxies
+ proxies=proxies,
)
response.raise_for_status()
prompt_tokens = 0
@@ -76,43 +88,45 @@ def _create_stream(
break
try:
data = json.loads(json_str)
- choices = data.get('choices')
+ choices = data.get("choices")
if not choices and choices is not None:
continue
choice_data = choices[0] if choices else {}
- delta_data = choice_data.get('delta', {})
- finish_reason = choice_data.get('finish_reason')
- usage_data = data.get('usage', {})
+ delta_data = choice_data.get("delta", {})
+ finish_reason = choice_data.get("finish_reason")
+ usage_data = data.get("usage", {})
if usage_data:
- prompt_tokens = usage_data.get('prompt_tokens', prompt_tokens)
- completion_tokens = usage_data.get('completion_tokens', completion_tokens)
- total_tokens = usage_data.get('total_tokens', total_tokens)
- if delta_data.get('content'):
+ prompt_tokens = usage_data.get("prompt_tokens", prompt_tokens)
+ completion_tokens = usage_data.get(
+ "completion_tokens", completion_tokens
+ )
+ total_tokens = usage_data.get("total_tokens", total_tokens)
+ if delta_data.get("content"):
completion_tokens += 1
total_tokens = prompt_tokens + completion_tokens
delta = ChoiceDelta(
- content=delta_data.get('content'),
- role=delta_data.get('role'),
- tool_calls=delta_data.get('tool_calls')
+ content=delta_data.get("content"),
+ role=delta_data.get("role"),
+ tool_calls=delta_data.get("tool_calls"),
)
choice = Choice(
- index=choice_data.get('index', 0),
+ index=choice_data.get("index", 0),
delta=delta,
finish_reason=finish_reason,
- logprobs=choice_data.get('logprobs')
+ logprobs=choice_data.get("logprobs"),
)
chunk = ChatCompletionChunk(
id=request_id,
choices=[choice],
created=created_time,
model=model,
- system_fingerprint=data.get('system_fingerprint')
+ system_fingerprint=data.get("system_fingerprint"),
)
chunk.usage = {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": total_tokens,
- "estimated_cost": None
+ "estimated_cost": None,
}
yield chunk
except json.JSONDecodeError:
@@ -125,13 +139,13 @@ def _create_stream(
choices=[choice],
created=created_time,
model=model,
- system_fingerprint=None
+ system_fingerprint=None,
)
chunk.usage = {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": total_tokens,
- "estimated_cost": None
+ "estimated_cost": None,
}
yield chunk
except Exception as e:
@@ -139,8 +153,13 @@ def _create_stream(
raise IOError(f"DeepInfra request failed: {e}") from e
def _create_non_stream(
- self, request_id: str, created_time: int, model: str, payload: Dict[str, Any],
- timeout: Optional[int] = None, proxies: Optional[Dict[str, str]] = None
+ self,
+ request_id: str,
+ created_time: int,
+ model: str,
+ payload: Dict[str, Any],
+ timeout: Optional[int] = None,
+ proxies: Optional[Dict[str, str]] = None,
) -> ChatCompletion:
try:
response = self._client.session.post(
@@ -148,43 +167,42 @@ def _create_non_stream(
headers=self._client.headers,
json=payload,
timeout=timeout or self._client.timeout,
- proxies=proxies
+ proxies=proxies,
)
response.raise_for_status()
data = response.json()
- choices_data = data.get('choices', [])
- usage_data = data.get('usage', {})
+ choices_data = data.get("choices", [])
+ usage_data = data.get("usage", {})
choices = []
for choice_d in choices_data:
- message_d = choice_d.get('message')
- if not message_d and 'delta' in choice_d:
- delta = choice_d['delta']
+ message_d = choice_d.get("message")
+ if not message_d and "delta" in choice_d:
+ delta = choice_d["delta"]
message_d = {
- 'role': delta.get('role', 'assistant'),
- 'content': delta.get('content', '')
+ "role": delta.get("role", "assistant"),
+ "content": delta.get("content", ""),
}
if not message_d:
- message_d = {'role': 'assistant', 'content': ''}
+ message_d = {"role": "assistant", "content": ""}
message = ChatCompletionMessage(
- role=message_d.get('role', 'assistant'),
- content=message_d.get('content', '')
+ role=message_d.get("role", "assistant"), content=message_d.get("content", "")
)
choice = Choice(
- index=choice_d.get('index', 0),
+ index=choice_d.get("index", 0),
message=message,
- finish_reason=choice_d.get('finish_reason', 'stop')
+ finish_reason=choice_d.get("finish_reason", "stop"),
)
choices.append(choice)
usage = CompletionUsage(
- prompt_tokens=usage_data.get('prompt_tokens', 0),
- completion_tokens=usage_data.get('completion_tokens', 0),
- total_tokens=usage_data.get('total_tokens', 0)
+ prompt_tokens=usage_data.get("prompt_tokens", 0),
+ completion_tokens=usage_data.get("completion_tokens", 0),
+ total_tokens=usage_data.get("total_tokens", 0),
)
completion = ChatCompletion(
id=request_id,
choices=choices,
created=created_time,
- model=data.get('model', model),
+ model=data.get("model", model),
usage=usage,
)
return completion
@@ -192,10 +210,12 @@ def _create_non_stream(
print(f"Error during DeepInfra non-stream request: {e}")
raise IOError(f"DeepInfra request failed: {e}") from e
+
class Chat(BaseChat):
- def __init__(self, client: 'DeepInfra'):
+ def __init__(self, client: "DeepInfra"):
self.completions = Completions(client)
+
class DeepInfra(OpenAICompatibleProvider):
required_auth = False
AVAILABLE_MODELS = [
@@ -279,7 +299,60 @@ class DeepInfra(OpenAICompatibleProvider):
"allenai/olmOCR-7B-0725-FP8",
]
+ @classmethod
+ def get_models(cls, api_key: Optional[str] = None):
+ """Fetch available models from DeepInfra API.
+
+ Args:
+ api_key (str, optional): DeepInfra API key. If not provided, returns default models.
+
+ Returns:
+ list: List of available model IDs
+ """
+ if not api_key:
+ return cls.AVAILABLE_MODELS
+
+ try:
+ # Use a temporary requests session for this class method
+ temp_session = requests.Session()
+ headers = {
+ "Content-Type": "application/json",
+ }
+ if api_key:
+ headers["Authorization"] = f"Bearer {api_key}"
+
+ response = temp_session.get(
+ "https://api.deepinfra.com/v1/models",
+ headers=headers,
+ )
+
+ if response.status_code != 200:
+ return cls.AVAILABLE_MODELS
+
+ data = response.json()
+ if "data" in data and isinstance(data["data"], list):
+ return [model["id"] for model in data["data"]]
+ return cls.AVAILABLE_MODELS
+
+ except Exception:
+ # Fallback to default models list if fetching fails
+ return cls.AVAILABLE_MODELS
+
+ @classmethod
+ def update_available_models(cls, api_key=None):
+ """Update the available models list from DeepInfra API"""
+ try:
+ models = cls.get_models(api_key)
+ if models and len(models) > 0:
+ cls.AVAILABLE_MODELS = models
+ except Exception:
+ # Fallback to default models list if fetching fails
+ pass
+
def __init__(self, browser: str = "chrome", api_key: str = None):
+ # Update available models from API
+ self.update_available_models(api_key)
+
self.timeout = None
self.base_url = "https://api.deepinfra.com/v1/openai/chat/completions"
self.session = requests.Session()
@@ -299,7 +372,8 @@ def __init__(self, browser: str = "chrome", api_key: str = None):
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-site",
"X-Deepinfra-Source": "web-embed",
- "Sec-CH-UA": fingerprint["sec_ch_ua"] or '"Not)A;Brand";v="99", "Microsoft Edge";v="127", "Chromium";v="127"',
+ "Sec-CH-UA": fingerprint["sec_ch_ua"]
+ or '"Not)A;Brand";v="99", "Microsoft Edge";v="127", "Chromium";v="127"',
"Sec-CH-UA-Mobile": "?0",
"Sec-CH-UA-Platform": f'"{fingerprint["platform"]}"',
"User-Agent": fingerprint["user_agent"],
@@ -308,19 +382,22 @@ def __init__(self, browser: str = "chrome", api_key: str = None):
self.headers["Authorization"] = f"Bearer {api_key}"
self.session.headers.update(self.headers)
self.chat = Chat(self)
+
@property
def models(self):
class _ModelList:
def list(inner_self):
- return type(self).AVAILABLE_MODELS
+ return DeepInfra.AVAILABLE_MODELS
+
return _ModelList()
+
if __name__ == "__main__":
client = DeepInfra()
response = client.chat.completions.create(
model="deepseek-ai/DeepSeek-R1-0528",
messages=[{"role": "user", "content": "Hello, how are you?"}],
max_tokens=10000,
- stream=False
+ stream=False,
)
- print(response.choices[0].message.content)
\ No newline at end of file
+ print(response.choices[0].message.content)
diff --git a/webscout/Provider/OPENAI/e2b.py b/webscout/Provider/OPENAI/e2b.py
index 38364245..fa258561 100644
--- a/webscout/Provider/OPENAI/e2b.py
+++ b/webscout/Provider/OPENAI/e2b.py
@@ -1627,7 +1627,6 @@ def create(
def _send_request(self, request_body: dict, model_config: dict, timeout: Optional[int] = None, proxies: Optional[Dict[str, str]] = None, retries: int = 3) -> str:
"""Enhanced request method with IP rotation, session rotation, and advanced rate limit bypass."""
url = model_config["apiUrl"]
- target_origin = "https://fragments.e2b.dev"
# Use client proxies if none provided
if proxies is None:
@@ -2171,13 +2170,16 @@ def _build_request_body(self, model_config: dict, messages: list, system_prompt:
def _merge_user_messages(self, messages: list) -> list:
"""Merges consecutive user messages"""
- if not messages: return []
+ if not messages:
+ return []
merged = []
current_message = messages[0]
for next_message in messages[1:]:
- if not isinstance(next_message, dict) or "role" not in next_message: continue
+ if not isinstance(next_message, dict) or "role" not in next_message:
+ continue
if not isinstance(current_message, dict) or "role" not in current_message:
- current_message = next_message; continue
+ current_message = next_message
+ continue
if current_message["role"] == "user" and next_message["role"] == "user":
if (isinstance(current_message.get("content"), list) and current_message["content"] and
isinstance(current_message["content"][0], dict) and current_message["content"][0].get("type") == "text" and
@@ -2185,23 +2187,32 @@ def _merge_user_messages(self, messages: list) -> list:
isinstance(next_message["content"][0], dict) and next_message["content"][0].get("type") == "text"):
current_message["content"][0]["text"] += "\n" + next_message["content"][0]["text"]
else:
- merged.append(current_message); current_message = next_message
+ merged.append(current_message)
+ current_message = next_message
else:
- merged.append(current_message); current_message = next_message
- if current_message not in merged: merged.append(current_message)
+ merged.append(current_message)
+ current_message = next_message
+ if current_message not in merged:
+ merged.append(current_message)
return merged
def _transform_content(self, messages: list) -> list:
"""Transforms message format and merges consecutive user messages"""
transformed = []
for msg in messages:
- if not isinstance(msg, dict): continue
+ if not isinstance(msg, dict):
+ continue
role, content = msg.get("role"), msg.get("content")
- if role is None or content is None: continue
- if isinstance(content, list): transformed.append(msg); continue
+ if role is None or content is None:
+ continue
+ if isinstance(content, list):
+ transformed.append(msg)
+ continue
if not isinstance(content, str):
- try: content = str(content)
- except Exception: continue
+ try:
+ content = str(content)
+ except Exception:
+ continue
base_content = {"type": "text", "text": content}
# System messages are handled separately now, no need for role-playing prompt here.
diff --git a/webscout/Provider/OPENAI/elmo.py b/webscout/Provider/OPENAI/elmo.py
index eb6a7859..c7e1031d 100644
--- a/webscout/Provider/OPENAI/elmo.py
+++ b/webscout/Provider/OPENAI/elmo.py
@@ -1,18 +1,20 @@
-import json
+import re
import time
import uuid
-import re
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
from curl_cffi.requests import Session
-from curl_cffi import CurlError
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.AIutel import sanitize_stream
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
)
-from webscout.AIutel import sanitize_stream
try:
from webscout.litagent import LitAgent
@@ -105,7 +107,7 @@ def _create_stream(
impersonate="chrome110"
)
response.raise_for_status()
-
+
# Use sanitize_stream to process the response
processed_stream = sanitize_stream(
data=response.iter_content(chunk_size=None), # Pass byte iterator
@@ -119,7 +121,7 @@ def _create_stream(
prompt_tokens = 0
completion_tokens = 0
total_tokens = 0
-
+
for content_chunk in processed_stream:
if content_chunk and isinstance(content_chunk, str):
completion_tokens += len(content_chunk.split())
@@ -149,7 +151,7 @@ def _create_stream(
"estimated_cost": None
}
yield chunk
-
+
# Final chunk
delta = ChoiceDelta(content=None, role=None, tool_calls=None)
choice = Choice(index=0, delta=delta, finish_reason="stop", logprobs=None)
@@ -185,7 +187,7 @@ def _create_non_stream(
proxies=proxies
)
response.raise_for_status()
-
+
# Use sanitize_stream to process the response and aggregate content
processed_stream = sanitize_stream(
data=response.iter_content(chunk_size=None), # Pass byte iterator
@@ -195,13 +197,13 @@ def _create_non_stream(
yield_raw_on_error=True,
raw=False
)
-
+
# Aggregate all content
content = ""
for content_chunk in processed_stream:
if content_chunk and isinstance(content_chunk, str):
content += content_chunk
-
+
message = ChatCompletionMessage(
role="assistant",
content=content
@@ -254,7 +256,7 @@ def __init__(self, browser: str = "chrome"):
}
self.session.headers.update(self.headers)
self.chat = Chat(self)
-
+
@property
def models(self):
class _ModelList:
@@ -270,4 +272,4 @@ def list(inner_self):
max_tokens=600,
stream=False
)
- print(response.choices[0].message.content)
\ No newline at end of file
+ print(response.choices[0].message.content)
diff --git a/webscout/Provider/OPENAI/exaai.py b/webscout/Provider/OPENAI/exaai.py
index fdfa31e3..00f8caeb 100644
--- a/webscout/Provider/OPENAI/exaai.py
+++ b/webscout/Provider/OPENAI/exaai.py
@@ -1,15 +1,21 @@
+import json
+import re
import time
import uuid
+from typing import Any, Dict, Generator, List, Optional, Union
+
import requests
-import json
-import re
-from typing import List, Dict, Optional, Union, Generator, Any
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ count_tokens,
)
# Attempt to import LitAgent, fallback if not available
@@ -434,7 +440,7 @@ def convert_model_name(self, model: str) -> str:
# ExaAI only supports O3-Mini, regardless of the input model
print(f"Note: ExaAI only supports O3-Mini model. Ignoring provided model '{model}'.")
return "O3-Mini"
-
+
@property
def models(self):
class _ModelList:
@@ -451,4 +457,4 @@ def list(inner_self):
{"role": "user", "content": "Hello, how are you?"}
]
)
- print(response.choices[0].message.content)
\ No newline at end of file
+ print(response.choices[0].message.content)
diff --git a/webscout/Provider/OPENAI/freeassist.py b/webscout/Provider/OPENAI/freeassist.py
index c7d7a4fa..90a207c8 100644
--- a/webscout/Provider/OPENAI/freeassist.py
+++ b/webscout/Provider/OPENAI/freeassist.py
@@ -1,13 +1,18 @@
-import requests
import json
import time
import uuid
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
+
+import requests
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
)
try:
@@ -35,7 +40,7 @@ def create(
) -> Union[ChatCompletion, Generator[ChatCompletionChunk, None, None]]:
"""
Create a chat completion with FreeAssist API.
-
+
Args:
model: Model identifier (e.g., 'google/gemini-2.5-flash-lite')
messages: List of message dictionaries with 'role' and 'content'
@@ -46,7 +51,7 @@ def create(
timeout: Request timeout
proxies: Proxy configuration
**kwargs: Additional parameters
-
+
Returns:
ChatCompletion or Generator of ChatCompletionChunk
"""
@@ -56,10 +61,10 @@ def create(
"anonymousUserId": str(uuid.uuid4()),
"isContinuation": False
}
-
+
request_id = f"chatcmpl-{uuid.uuid4()}"
created_time = int(time.time())
-
+
if stream:
return self._create_stream(request_id, created_time, model, payload, timeout, proxies)
else:
@@ -80,52 +85,52 @@ def _create_stream(
proxies=proxies,
)
response.raise_for_status()
-
+
prompt_tokens = 0
completion_tokens = 0
total_tokens = 0
-
+
for line in response.iter_lines(decode_unicode=True):
if not line:
continue
-
+
if isinstance(line, bytes):
try:
line = line.decode("utf-8")
except Exception:
continue
-
+
if line.startswith("data: "):
json_str = line[6:]
else:
json_str = line
-
+
if json_str == "[DONE]":
break
-
+
try:
data = json.loads(json_str)
-
+
# Extract usage if present
usage_data = data.get('usage', {})
if usage_data:
prompt_tokens = usage_data.get('prompt_tokens', prompt_tokens)
completion_tokens = usage_data.get('completion_tokens', completion_tokens)
total_tokens = usage_data.get('total_tokens', total_tokens)
-
+
choices = data.get('choices')
if not choices and choices is not None:
continue
-
+
choice_data = choices[0] if choices else {}
delta_data = choice_data.get('delta', {})
finish_reason = choice_data.get('finish_reason')
-
+
# Get content
content_piece = None
role = None
tool_calls = None
-
+
if delta_data:
content_piece = delta_data.get('content')
role = delta_data.get('role')
@@ -135,24 +140,24 @@ def _create_stream(
role = message_d.get("role")
content_piece = message_d.get("content")
tool_calls = message_d.get("tool_calls")
-
+
if content_piece and not usage_data:
completion_tokens += 1
total_tokens = prompt_tokens + completion_tokens
-
+
delta = ChoiceDelta(
content=content_piece,
role=role,
tool_calls=tool_calls
)
-
+
choice = Choice(
index=choice_data.get('index', 0),
delta=delta,
finish_reason=finish_reason,
logprobs=choice_data.get('logprobs')
)
-
+
chunk = ChatCompletionChunk(
id=data.get('id', request_id),
choices=[choice],
@@ -160,19 +165,19 @@ def _create_stream(
model=data.get('model', model),
system_fingerprint=data.get('system_fingerprint')
)
-
+
chunk.usage = {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": total_tokens,
"estimated_cost": None
}
-
+
yield chunk
-
+
except json.JSONDecodeError:
continue
-
+
# Final chunk with finish_reason="stop"
delta = ChoiceDelta(content=None, role=None, tool_calls=None)
choice = Choice(index=0, delta=delta, finish_reason="stop", logprobs=None)
@@ -190,7 +195,7 @@ def _create_stream(
"estimated_cost": None
}
yield chunk
-
+
except Exception as e:
print(f"Error during FreeAssist stream request: {e}")
raise IOError(f"FreeAssist request failed: {e}") from e
@@ -211,75 +216,75 @@ def _create_non_stream(
proxies=proxies,
)
response.raise_for_status()
-
+
full_content = ""
prompt_tokens = 0
completion_tokens = 0
total_tokens = 0
response_model = model
-
+
for line in response.iter_lines(decode_unicode=True):
if not line:
continue
-
+
if isinstance(line, bytes):
try:
line = line.decode("utf-8")
except Exception:
continue
-
+
if line.startswith("data: "):
json_str = line[6:]
else:
json_str = line
-
+
if json_str == "[DONE]":
break
-
+
try:
data = json.loads(json_str)
-
+
# Extract usage if present
usage_data = data.get('usage', {})
if usage_data:
prompt_tokens = usage_data.get('prompt_tokens', prompt_tokens)
completion_tokens = usage_data.get('completion_tokens', completion_tokens)
total_tokens = usage_data.get('total_tokens', total_tokens)
-
+
choices = data.get('choices')
if not choices and choices is not None:
continue
-
+
choice_data = choices[0] if choices else {}
delta_data = choice_data.get('delta', {})
content = delta_data.get('content', '')
-
+
if content:
full_content += content
-
+
# Get model from response
if data.get('model'):
response_model = data.get('model')
except json.JSONDecodeError:
continue
-
+
message = ChatCompletionMessage(
role='assistant',
content=full_content
)
-
+
choice = Choice(
index=0,
message=message,
finish_reason='stop'
)
-
+
usage = CompletionUsage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=total_tokens
)
-
+
completion = ChatCompletion(
id=request_id,
choices=[choice],
@@ -287,9 +292,9 @@ def _create_non_stream(
model=response_model,
usage=usage,
)
-
+
return completion
-
+
except Exception as e:
print(f"Error during FreeAssist non-stream request: {e}")
raise IOError(f"FreeAssist request failed: {e}") from e
@@ -303,15 +308,15 @@ def __init__(self, client: 'FreeAssist'):
class FreeAssist(OpenAICompatibleProvider):
"""
FreeAssist - A free OpenAI-compatible provider using FreeAssist.ai
-
+
This provider uses the FreeAssist API which provides access to various
AI models including Google's Gemini series.
-
+
Usage:
from webscout.Provider.OPENAI import FreeAssist
-
+
client = FreeAssist()
-
+
# Streaming
for chunk in client.chat.completions.create(
model="google/gemini-2.5-flash-lite",
@@ -320,7 +325,7 @@ class FreeAssist(OpenAICompatibleProvider):
):
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="", flush=True)
-
+
# Non-streaming
response = client.chat.completions.create(
model="google/gemini-2.5-flash-lite",
@@ -329,7 +334,7 @@ class FreeAssist(OpenAICompatibleProvider):
)
print(response.choices[0].message.content)
"""
-
+
AVAILABLE_MODELS = [
"google/gemini-2.5-flash-lite",
"google/gemini-2.5-flash",
@@ -345,7 +350,7 @@ def __init__(
):
"""
Initialize the FreeAssist provider.
-
+
Args:
browser: Browser to impersonate for fingerprinting
timeout: Request timeout in seconds
@@ -354,10 +359,10 @@ def __init__(
self.timeout = timeout
self.base_url = "https://qcpujeurnkbvwlvmylyx.supabase.co/functions/v1/chat"
self.session = requests.Session()
-
+
agent = LitAgent()
fingerprint = agent.generate_fingerprint(browser)
-
+
self.headers = {
'accept': '*/*',
'accept-language': 'en-US,en;q=0.9,en-IN;q=0.8',
@@ -374,13 +379,13 @@ def __init__(
'sec-gpc': '1',
'user-agent': fingerprint.get("user_agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36 Edg/143.0.0.0")
}
-
+
self.session.headers.update(self.headers)
if proxies:
self.session.proxies.update(proxies)
-
+
self.chat = Chat(self)
-
+
@property
def models(self):
"""Return available models."""
@@ -394,9 +399,9 @@ def list(inner_self):
print("-" * 80)
print("Testing FreeAssist Provider")
print("-" * 80)
-
+
client = FreeAssist()
-
+
# Test streaming
print("\n[Streaming Test]")
try:
@@ -412,7 +417,7 @@ def list(inner_self):
print(f"\n✓ Streaming works! Response: {response_text}")
except Exception as e:
print(f"✗ Streaming failed: {e}")
-
+
# Test non-streaming
print("\n[Non-Streaming Test]")
try:
@@ -424,6 +429,6 @@ def list(inner_self):
print(f"✓ Non-streaming works! Response: {response.choices[0].message.content}")
except Exception as e:
print(f"✗ Non-streaming failed: {e}")
-
+
print("\n" + "-" * 80)
print("Available models:", FreeAssist.AVAILABLE_MODELS)
diff --git a/webscout/Provider/OPENAI/gradient.py b/webscout/Provider/OPENAI/gradient.py
index c9bafcfc..7fc46508 100644
--- a/webscout/Provider/OPENAI/gradient.py
+++ b/webscout/Provider/OPENAI/gradient.py
@@ -8,13 +8,19 @@
import json
import time
import uuid
+from typing import Any, Dict, Generator, List, Optional, Union
+
import requests
-from typing import List, Dict, Optional, Union, Generator, Any
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ count_tokens,
)
# ANSI escape codes for formatting
@@ -67,7 +73,7 @@ def create(
# Convert model name and get appropriate cluster mode
converted_model = self._client.convert_model_name(model)
actual_cluster_mode = cluster_mode or self._client.MODEL_CLUSTERS.get(converted_model, self._client.cluster_mode)
-
+
# Build the payload - pass messages directly as the API accepts them
payload = {
"model": converted_model,
@@ -117,32 +123,32 @@ def _create_stream(
# Parse JSON response
data = json.loads(decoded_line)
-
+
# Only process "reply" type chunks
chunk_type = data.get("type")
if chunk_type != "reply":
continue
-
+
# Extract content - prefer "content" over "reasoningContent"
reply_data = data.get("data", {})
content = reply_data.get("content") or reply_data.get("reasoningContent")
-
+
if content:
completion_tokens += count_tokens(content)
-
+
delta = ChoiceDelta(
content=content,
role="assistant" if first_chunk else None
)
first_chunk = False
-
+
choice = Choice(
index=0,
delta=delta,
finish_reason=None,
logprobs=None
)
-
+
chunk = ChatCompletionChunk(
id=request_id,
choices=[choice],
@@ -197,7 +203,7 @@ def _create_non_stream(
# Collect all chunks from streaming
full_content = ""
prompt_tokens = count_tokens(str(payload.get("messages", [])))
-
+
response = self._client.session.post(
self._client.base_url,
headers=self._client.headers,
@@ -220,12 +226,12 @@ def _create_non_stream(
continue
data = json.loads(decoded_line)
-
+
# Only process "reply" type chunks
chunk_type = data.get("type")
if chunk_type != "reply":
continue
-
+
reply_data = data.get("data", {})
# Prefer "content" over "reasoningContent"
content = reply_data.get("content") or reply_data.get("reasoningContent")
@@ -285,10 +291,10 @@ def __init__(self, client: 'Gradient'):
class Gradient(OpenAICompatibleProvider):
"""
OpenAI-compatible client for Gradient Network API.
-
+
Gradient Network provides access to distributed GPU clusters running large language models.
This provider supports real-time streaming responses.
-
+
Note: GPT OSS 120B works on "nvidia" cluster, Qwen3 235B works on "hybrid" cluster.
Cluster mode is auto-detected based on model selection.
@@ -316,7 +322,7 @@ class Gradient(OpenAICompatibleProvider):
"GPT OSS 120B",
"Qwen3 235B",
]
-
+
# Model to cluster mapping
MODEL_CLUSTERS = {
"GPT OSS 120B": "nvidia",
@@ -343,7 +349,7 @@ def __init__(
self.cluster_mode = cluster_mode
self.enable_thinking = enable_thinking
self.proxies = proxies or {}
-
+
self.base_url = "https://chat.gradient.network/api/generate"
self.session = requests.Session()
diff --git a/webscout/Provider/OPENAI/groq.py b/webscout/Provider/OPENAI/groq.py
index b16dda2f..803d7917 100644
--- a/webscout/Provider/OPENAI/groq.py
+++ b/webscout/Provider/OPENAI/groq.py
@@ -1,18 +1,22 @@
-import requests
import json
import time
import uuid
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
+
+from curl_cffi import CurlError
# Import curl_cffi for improved request handling
from curl_cffi.requests import Session
-from curl_cffi import CurlError
# Import base classes and utility structures
-from .base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from .base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from .utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
)
# Attempt to import LitAgent, fallback if not available
@@ -58,7 +62,7 @@ def create(
payload["frequency_penalty"] = kwargs.pop("frequency_penalty")
if "presence_penalty" in kwargs:
payload["presence_penalty"] = kwargs.pop("presence_penalty")
-
+
# Add any tools if provided
if "tools" in kwargs and kwargs["tools"]:
payload["tools"] = kwargs.pop("tools")
@@ -84,7 +88,7 @@ def _create_stream(
timeout=self._client.timeout,
impersonate="chrome110" # Use impersonate for better compatibility
)
-
+
if response.status_code != 200:
raise IOError(f"Groq request failed with status code {response.status_code}: {response.text}")
@@ -184,10 +188,10 @@ def _create_non_stream(
timeout=self._client.timeout,
impersonate="chrome110" # Use impersonate for better compatibility
)
-
+
if response.status_code != 200:
raise IOError(f"Groq request failed with status code {response.status_code}: {response.text}")
-
+
data = response.json()
choices_data = data.get('choices', [])
@@ -196,10 +200,10 @@ def _create_non_stream(
choices = []
for choice_d in choices_data:
message_d = choice_d.get('message', {})
-
+
# Handle tool calls if present
tool_calls = message_d.get('tool_calls')
-
+
message = ChatCompletionMessage(
role=message_d.get('role', 'assistant'),
content=message_d.get('content', ''),
@@ -272,26 +276,26 @@ def __init__(self, api_key: str = None, timeout: Optional[int] = 30, browser: st
self.timeout = timeout
self.base_url = "https://api.groq.com/openai/v1/chat/completions"
self.api_key = api_key
-
+
# Update available models from API
self.update_available_models(api_key)
-
+
# Initialize curl_cffi Session
self.session = Session()
-
+
# Set up headers with API key if provided
self.headers = {
"Content-Type": "application/json",
}
-
+
if api_key:
self.headers["Authorization"] = f"Bearer {api_key}"
-
+
# Try to use LitAgent for browser fingerprinting
try:
agent = LitAgent()
fingerprint = agent.generate_fingerprint(browser)
-
+
self.headers.update({
"Accept": fingerprint["accept"],
"Accept-Encoding": "gzip, deflate, br, zstd",
@@ -317,26 +321,26 @@ def __init__(self, api_key: str = None, timeout: Optional[int] = 30, browser: st
"Accept-Language": "en-US,en;q=0.9",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
})
-
+
# Update session headers
self.session.headers.update(self.headers)
-
+
# Initialize chat interface
self.chat = Chat(self)
-
+
@classmethod
def get_models(cls, api_key: str = None):
"""Fetch available models from Groq API.
-
+
Args:
api_key (str, optional): Groq API key. If not provided, returns default models.
-
+
Returns:
list: List of available model IDs
"""
if not api_key:
return cls.AVAILABLE_MODELS
-
+
try:
# Use a temporary curl_cffi session for this class method
temp_session = Session()
@@ -344,21 +348,21 @@ def get_models(cls, api_key: str = None):
"Content-Type": "application/json",
"Authorization": f"Bearer {api_key}",
}
-
+
response = temp_session.get(
"https://api.groq.com/openai/v1/models",
headers=headers,
impersonate="chrome110" # Use impersonate for fetching
)
-
+
if response.status_code != 200:
return cls.AVAILABLE_MODELS
-
+
data = response.json()
if "data" in data and isinstance(data["data"], list):
return [model["id"] for model in data["data"]]
return cls.AVAILABLE_MODELS
-
+
except (CurlError, Exception):
# Fallback to default models list if fetching fails
return cls.AVAILABLE_MODELS
diff --git a/webscout/Provider/OPENAI/hadadxyz.py b/webscout/Provider/OPENAI/hadadxyz.py
index 07803748..1bb4319e 100644
--- a/webscout/Provider/OPENAI/hadadxyz.py
+++ b/webscout/Provider/OPENAI/hadadxyz.py
@@ -1,17 +1,20 @@
-import json
import time
import uuid
-import re
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
from curl_cffi.requests import Session
-from curl_cffi import CurlError
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, count_tokens, format_prompt
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ count_tokens,
+ format_prompt,
)
try:
@@ -69,10 +72,10 @@ def create(
include_think_tags: bool = True,
**kwargs: Any
) -> Union[ChatCompletion, Generator[ChatCompletionChunk, None, None]]:
-
+
# Format the prompt using the utility
prompt = format_prompt(messages, include_system=True)
-
+
payload = {
"tools": {},
"modelId": model,
@@ -93,7 +96,7 @@ def create(
request_id = f"chatcmpl-{uuid.uuid4()}"
created_time = int(time.time())
-
+
if stream:
return self._create_stream(request_id, created_time, model, payload, timeout, proxies, include_think_tags)
else:
@@ -105,7 +108,7 @@ def _create_stream(
include_think_tags: bool = True
) -> Generator[ChatCompletionChunk, None, None]:
extractor = _DeltaExtractor(include_think_tags=include_think_tags)
-
+
try:
response = self._client.session.post(
self._client.api_endpoint,
@@ -117,10 +120,10 @@ def _create_stream(
impersonate="chrome120"
)
response.raise_for_status()
-
+
prompt_tokens = count_tokens(payload["messages"][0]["parts"][0]["text"])
completion_tokens = 0
-
+
from webscout.AIutel import sanitize_stream
processed_stream = sanitize_stream(
data=response.iter_lines(),
@@ -134,7 +137,7 @@ def _create_stream(
for content_chunk in processed_stream:
if content_chunk and isinstance(content_chunk, str):
completion_tokens += count_tokens(content_chunk)
-
+
delta = ChoiceDelta(
content=content_chunk,
role="assistant"
@@ -180,14 +183,14 @@ def _create_non_stream(
) -> ChatCompletion:
full_content = ""
prompt_tokens = count_tokens(payload["messages"][0]["parts"][0]["text"])
-
+
try:
for chunk in self._create_stream(request_id, created_time, model, payload, timeout, proxies, include_think_tags):
if chunk.choices and chunk.choices[0].delta.content:
full_content += chunk.choices[0].delta.content
-
+
completion_tokens = count_tokens(full_content)
-
+
message = ChatCompletionMessage(
role="assistant",
content=full_content
diff --git a/webscout/Provider/OPENAI/heckai.py b/webscout/Provider/OPENAI/heckai.py
index 631813ae..2fa9e4df 100644
--- a/webscout/Provider/OPENAI/heckai.py
+++ b/webscout/Provider/OPENAI/heckai.py
@@ -1,19 +1,20 @@
import time
import uuid
+from typing import Any, Dict, Generator, List, Optional, Union
+
import requests
-from typing import List, Dict, Optional, Union, Generator, Any
from webscout.litagent import LitAgent
from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
ChatCompletion,
ChatCompletionChunk,
- Choice,
ChatCompletionMessage,
+ Choice,
ChoiceDelta,
CompletionUsage,
+ count_tokens,
format_prompt,
- count_tokens
)
# ANSI escape codes for formatting
diff --git a/webscout/Provider/OPENAI/huggingface.py b/webscout/Provider/OPENAI/huggingface.py
index 539f1d6f..0848cb6b 100644
--- a/webscout/Provider/OPENAI/huggingface.py
+++ b/webscout/Provider/OPENAI/huggingface.py
@@ -1,15 +1,19 @@
import json
import time
import uuid
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
from curl_cffi.requests import Session
-from curl_cffi import CurlError
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ count_tokens,
)
try:
@@ -45,10 +49,10 @@ def create(
if top_p is not None:
payload["top_p"] = top_p
payload.update(kwargs)
-
+
request_id = f"chatcmpl-{uuid.uuid4()}"
created_time = int(time.time())
-
+
if stream:
return self._create_stream(request_id, created_time, model, payload, timeout, proxies)
else:
@@ -70,7 +74,7 @@ def _create_stream(
proxies=proxies
)
response.raise_for_status()
-
+
prompt_tokens = count_tokens([msg.get("content", "") for msg in payload.get("messages", [])])
completion_tokens = 0
total_tokens = 0
@@ -103,7 +107,7 @@ def _create_stream(
if content:
completion_tokens += count_tokens(content)
total_tokens = prompt_tokens + completion_tokens
-
+
delta = ChoiceDelta(
content=content,
role=delta_data.get('role'),
@@ -131,7 +135,7 @@ def _create_stream(
yield chunk
except json.JSONDecodeError:
continue
-
+
# Final chunk with finish_reason="stop"
delta = ChoiceDelta(content=None, role=None, tool_calls=None)
choice = Choice(index=0, delta=delta, finish_reason="stop", logprobs=None)
@@ -149,7 +153,7 @@ def _create_stream(
"estimated_cost": None
}
yield chunk
-
+
except Exception as e:
raise IOError(f"HuggingFace stream request failed: {e}") from e
@@ -168,10 +172,10 @@ def _create_non_stream(
)
response.raise_for_status()
data = response.json()
-
+
choices_data = data.get('choices', [])
usage_data = data.get('usage', {})
-
+
choices = []
for choice_d in choices_data:
message_d = choice_d.get('message', {})
@@ -185,13 +189,13 @@ def _create_non_stream(
finish_reason=choice_d.get('finish_reason', 'stop')
)
choices.append(choice)
-
+
usage = CompletionUsage(
prompt_tokens=usage_data.get('prompt_tokens', 0),
completion_tokens=usage_data.get('completion_tokens', 0),
total_tokens=usage_data.get('total_tokens', 0)
)
-
+
completion = ChatCompletion(
id=data.get('id', request_id),
choices=choices,
@@ -210,7 +214,7 @@ def __init__(self, client: 'HuggingFace'):
class HuggingFace(OpenAICompatibleProvider):
"""
OpenAI-compatible client for Hugging Face Inference API.
-
+
Requires an API key from https://huggingface.co/settings/tokens
"""
required_auth = True
@@ -226,7 +230,7 @@ def get_models(cls, api_key: str = None) -> List[str]:
headers = {}
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
-
+
response = temp_session.get(url, headers=headers, timeout=10)
if response.status_code == 200:
data = response.json()
@@ -251,7 +255,7 @@ def update_available_models(cls, api_key: str = None):
def __init__(self, api_key: str, browser: str = "chrome", timeout: int = 30):
if not api_key:
raise ValueError("API key is required for HuggingFace")
-
+
# Update available models from API
self.update_available_models(api_key)
@@ -259,10 +263,10 @@ def __init__(self, api_key: str, browser: str = "chrome", timeout: int = 30):
self.timeout = timeout
self.base_url = "https://router.huggingface.co/v1/chat/completions"
self.session = Session()
-
+
agent = LitAgent()
fingerprint = agent.generate_fingerprint(browser)
-
+
self.headers = {
"Accept": fingerprint["accept"],
"Accept-Language": fingerprint["accept_language"],
@@ -291,4 +295,4 @@ def list(inner_self):
# messages=[{"role": "user", "content": "Hello!"}]
# )
# print(response.choices[0].message.content)
- pass
\ No newline at end of file
+ pass
diff --git a/webscout/Provider/OPENAI/ibm.py b/webscout/Provider/OPENAI/ibm.py
index 8cb793b5..a474a4ee 100644
--- a/webscout/Provider/OPENAI/ibm.py
+++ b/webscout/Provider/OPENAI/ibm.py
@@ -1,19 +1,25 @@
import json
import time
import uuid
-from typing import List, Dict, Optional, Union, Generator, Any
from datetime import datetime
+from typing import Any, Dict, Generator, List, Optional, Union
+
+from curl_cffi import CurlError
# Import curl_cffi for improved request handling
from curl_cffi.requests import Session
-from curl_cffi import CurlError
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage,
- format_prompt, count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ count_tokens,
+ format_prompt,
)
# Attempt to import LitAgent, fallback if not available
@@ -43,12 +49,12 @@ def create(
Mimics openai.chat.completions.create
"""
formatted_prompt = format_prompt(
- messages,
- add_special_tokens=False,
+ messages,
+ add_special_tokens=False,
do_continue=True,
include_system=True
)
-
+
if not formatted_prompt:
raise ValueError("No valid prompt could be generated from messages")
@@ -101,7 +107,7 @@ def _create_stream(
timeout=self._client.timeout,
impersonate="chrome110"
)
-
+
if response.status_code in [401, 403]:
# Token expired, refresh and retry once
self._client.get_token()
@@ -123,56 +129,56 @@ def _create_stream(
for chunk in response.iter_content(chunk_size=None):
if not chunk:
continue
-
+
# Decode bytes to string
try:
chunk_str = chunk.decode('utf-8') if isinstance(chunk, bytes) else chunk
except UnicodeDecodeError:
continue
-
+
buffer += chunk_str
-
+
# Process complete lines
while '\n' in buffer:
line, buffer = buffer.split('\n', 1)
line = line.strip()
-
+
if not line:
continue
-
+
# Parse SSE format: "data: {...}"
if line.startswith("data:"):
json_str = line[5:].strip() # Remove "data:" prefix
-
+
# Skip [DONE] marker
if json_str == "[DONE]":
break
-
+
try:
# Parse JSON
data = json.loads(json_str)
-
+
# Extract content from IBM format
if data.get("type") == "message.part":
part = data.get("part", {})
content = part.get("content")
-
+
if content:
completion_tokens += 1
-
+
# Create the delta object
delta = ChoiceDelta(
content=content,
role="assistant"
)
-
+
# Create the choice object
choice = Choice(
index=0,
delta=delta,
finish_reason=None
)
-
+
# Create the chunk object
chunk = ChatCompletionChunk(
id=request_id,
@@ -181,9 +187,9 @@ def _create_stream(
model=model,
system_fingerprint=None
)
-
+
yield chunk
-
+
except json.JSONDecodeError:
# Skip malformed JSON lines
continue
@@ -309,7 +315,7 @@ def get_token(self) -> str:
def __init__(self, api_key: str = None, timeout: Optional[int] = 30, browser: str = "chrome"):
"""
Initialize IBM client.
-
+
Args:
api_key: Not required for IBM Granite Playground (uses dynamic bearer token)
timeout: Request timeout in seconds
@@ -317,15 +323,15 @@ def __init__(self, api_key: str = None, timeout: Optional[int] = 30, browser: st
"""
self.timeout = timeout
self.base_url = "https://d1eh1ubv87xmm5.cloudfront.net/granite/playground/api/v1/acp/runs"
-
+
# Initialize curl_cffi Session
self.session = Session()
-
+
# Initialize LitAgent for browser fingerprinting
try:
agent = LitAgent()
fingerprint = agent.generate_fingerprint(browser)
-
+
self.headers = {
"Accept": "text/event-stream",
"Accept-Language": fingerprint.get("accept_language", "en-US,en;q=0.9"),
@@ -362,23 +368,23 @@ def __init__(self, api_key: str = None, timeout: Optional[int] = 30, browser: st
"Sec-CH-UA-Mobile": "?0",
"Sec-CH-UA-Platform": '"Windows"',
}
-
+
# Update session headers
self.session.headers.update(self.headers)
# Fetch initial token
self.get_token()
-
+
# Initialize chat interface
self.chat = Chat(self)
@classmethod
def get_models(cls, api_key: str = None):
"""Get available models.
-
+
Args:
api_key: Not used for IBM (kept for compatibility)
-
+
Returns:
list: List of available model IDs
"""
@@ -397,7 +403,7 @@ def list(inner_self):
if __name__ == "__main__":
# Test the IBM client
client = IBM()
-
+
# Test streaming
print("Testing streaming:")
response = client.chat.completions.create(
@@ -407,12 +413,12 @@ def list(inner_self):
],
stream=True
)
-
+
for chunk in response:
if chunk.choices and chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="", flush=True)
print("\n")
-
+
# Test non-streaming
print("Testing non-streaming:")
response = client.chat.completions.create(
@@ -422,5 +428,5 @@ def list(inner_self):
],
stream=False
)
-
+
print(response.choices[0].message.content)
diff --git a/webscout/Provider/OPENAI/llmchat.py b/webscout/Provider/OPENAI/llmchat.py
index f7613bfd..65f135a0 100644
--- a/webscout/Provider/OPENAI/llmchat.py
+++ b/webscout/Provider/OPENAI/llmchat.py
@@ -1,21 +1,23 @@
+import json
import time
import uuid
-import json
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
from curl_cffi.requests import Session
-from curl_cffi import CurlError
+# Import LitAgent for user agent generation
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ count_tokens,
)
-from webscout.Provider.OPENAI.utils import format_prompt
-# Import LitAgent for user agent generation
-from webscout.litagent import LitAgent
class Completions(BaseCompletions):
def __init__(self, client: 'LLMChat'):
@@ -34,7 +36,7 @@ def create(
proxies: Optional[dict] = None,
**kwargs: Any
) -> Union[ChatCompletion, Generator[ChatCompletionChunk, None, None]]:
-
+
# In this case, we pass messages directly to the API
request_id = f"chatcmpl-{uuid.uuid4()}"
created_time = int(time.time())
@@ -57,7 +59,7 @@ def _create_streaming(
try:
prompt_tokens = count_tokens(json.dumps(messages))
completion_tokens = 0
-
+
url = f"{self._client.api_endpoint}?model={model}"
payload = {
"messages": messages,
@@ -82,14 +84,14 @@ def _create_streaming(
data_str = line[6:]
if data_str.strip() == "[DONE]":
break
-
+
try:
data = json.loads(data_str)
content = data.get('response', '')
if content:
full_content += content
completion_tokens += 1
-
+
delta = ChoiceDelta(content=content, role="assistant")
choice = Choice(index=0, delta=delta, finish_reason=None)
chunk = ChatCompletionChunk(
@@ -128,11 +130,11 @@ def _create_non_streaming(
try:
full_content = ""
prompt_tokens = count_tokens(json.dumps(messages))
-
+
for chunk in self._create_streaming(request_id, created_time, model, messages, max_tokens, timeout, proxies):
if chunk.choices[0].delta.content:
full_content += chunk.choices[0].delta.content
-
+
message = ChatCompletionMessage(role="assistant", content=full_content)
choice = Choice(index=0, message=message, finish_reason="stop")
usage = CompletionUsage(
@@ -140,7 +142,7 @@ def _create_non_streaming(
completion_tokens=count_tokens(full_content),
total_tokens=prompt_tokens + count_tokens(full_content)
)
-
+
return ChatCompletion(
id=request_id,
choices=[choice],
@@ -213,7 +215,7 @@ def __init__(self, proxies: dict = {}, timeout: int = 30):
self.api_endpoint = "https://llmchat.in/inference/stream"
self.proxies = proxies
self.session.proxies = proxies
-
+
self.headers = {
"Content-Type": "application/json",
"Accept": "*/*",
diff --git a/webscout/Provider/OPENAI/llmchatco.py b/webscout/Provider/OPENAI/llmchatco.py
index efee0b5b..b80beea4 100644
--- a/webscout/Provider/OPENAI/llmchatco.py
+++ b/webscout/Provider/OPENAI/llmchatco.py
@@ -1,14 +1,21 @@
+import json
import time
import uuid
+from typing import Any, Dict, Generator, List, Optional, Union
+
import requests
-import json
-from typing import List, Dict, Optional, Union, Generator, Any
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, get_last_user_message, get_system_prompt, format_prompt # Import format_prompt
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage, # Import format_prompt
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ get_last_user_message,
+ get_system_prompt,
)
# Attempt to import LitAgent, fallback if not available
@@ -52,7 +59,7 @@ def create(
# Determine the effective system prompt
effective_system_prompt = system_prompt # Use the provided system_prompt or its default
- message_list_system_prompt = get_system_prompt(messages)
+ get_system_prompt(messages)
# If a system prompt is also in messages, the explicit one takes precedence.
# We'll use the effective_system_prompt determined above.
@@ -344,4 +351,4 @@ def list(inner_self):
messages=[{"role": "user", "content": "Hello, how are you?"}],
stream=False
)
- print(response.choices[0].message.content)
\ No newline at end of file
+ print(response.choices[0].message.content)
diff --git a/webscout/Provider/OPENAI/meta.py b/webscout/Provider/OPENAI/meta.py
index 9b853e7f..888f424c 100644
--- a/webscout/Provider/OPENAI/meta.py
+++ b/webscout/Provider/OPENAI/meta.py
@@ -7,15 +7,19 @@
import time
import urllib
import uuid
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
-from curl_cffi.requests import Session
from curl_cffi import CurlError
+from curl_cffi.requests import Session
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
)
try:
@@ -176,7 +180,7 @@ def _create_stream(
try:
json_line = json.loads(line)
message_text = format_response(json_line)
-
+
if not message_text:
continue
@@ -304,7 +308,7 @@ def _create_non_stream(
raw_response = response.text
last_response = None
full_message = ""
-
+
for line in raw_response.split("\n"):
try:
json_line = json.loads(line)
@@ -363,7 +367,7 @@ class Meta(OpenAICompatibleProvider):
No API key required - uses web authentication.
"""
required_auth = False
-
+
AVAILABLE_MODELS = [
"meta-ai",
"llama-3"
@@ -379,7 +383,7 @@ def __init__(
):
"""
Initialize the Meta AI OpenAI-compatible client.
-
+
Args:
fb_email: Optional Facebook email for authenticated access
fb_password: Optional Facebook password for authenticated access
@@ -391,30 +395,30 @@ def __init__(
self.fb_password = fb_password
self.timeout = timeout
self.proxies = proxies or {}
-
+
self.session = Session()
if LitAgent:
agent = LitAgent()
self.session.headers.update({
"user-agent": agent.random()
})
-
+
self.access_token = None
self.is_authed = fb_password is not None and fb_email is not None
self.cookies = self._get_cookies()
self.external_conversation_id = None
self.offline_threading_id = None
-
+
if proxies:
self.session.proxies = proxies
-
+
# Initialize chat interface
self.chat = Chat(self)
def _get_cookies(self) -> dict:
"""Extracts necessary cookies from the Meta AI main page."""
headers = {}
-
+
# Import Facebook login if needed
if self.fb_email is not None and self.fb_password is not None:
try:
@@ -513,7 +517,7 @@ def list(inner_self):
# Example usage - no API key required
client = Meta()
print(f"Available models: {client.models.list()}")
-
+
# Test non-streaming
response = client.chat.completions.create(
model="meta-ai",
@@ -521,7 +525,7 @@ def list(inner_self):
stream=False
)
print(f"Response: {response.choices[0].message.content}")
-
+
# Test streaming
print("\nStreaming response:")
for chunk in client.chat.completions.create(
diff --git a/webscout/Provider/OPENAI/netwrck.py b/webscout/Provider/OPENAI/netwrck.py
index 9df1d11d..4bb81b9f 100644
--- a/webscout/Provider/OPENAI/netwrck.py
+++ b/webscout/Provider/OPENAI/netwrck.py
@@ -1,21 +1,22 @@
+import json
import time
import uuid
+from typing import Any, Dict, Generator, List, Optional, Union
+
import requests
-import json
-from typing import List, Dict, Optional, Union, Generator, Any
from webscout.litagent import LitAgent
from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
ChatCompletion,
ChatCompletionChunk,
- Choice,
ChatCompletionMessage,
+ Choice,
ChoiceDelta,
CompletionUsage,
+ count_tokens,
format_prompt,
get_system_prompt,
- count_tokens
)
# ANSI escape codes for formatting
@@ -47,8 +48,8 @@ def create(
# Format the messages using the format_prompt utility
# This creates a conversation in the format: "User: message\nAssistant: response\nUser: message\nAssistant:"
formatted_prompt = format_prompt(messages, add_special_tokens=True, do_continue=True)
-
-
+
+
# Prepare the payload for Netwrck API
payload = {
"query": formatted_prompt,
@@ -87,7 +88,7 @@ def _create_stream(
for line in response.iter_lines():
if not line:
continue
-
+
try:
decoded_line = line.decode('utf-8').strip('"')
if decoded_line:
@@ -95,18 +96,18 @@ def _create_stream(
formatted_content = self._client.format_text(decoded_line)
streaming_text += formatted_content
completion_tokens += count_tokens(formatted_content)
-
+
# Create a delta object for this chunk
delta = ChoiceDelta(content=formatted_content)
choice = Choice(index=0, delta=delta, finish_reason=None)
-
+
chunk = ChatCompletionChunk(
id=request_id,
choices=[choice],
created=created_time,
model=model,
)
-
+
yield chunk
except Exception:
continue
@@ -114,14 +115,14 @@ def _create_stream(
# Final chunk with finish_reason
delta = ChoiceDelta(content=None)
choice = Choice(index=0, delta=delta, finish_reason="stop")
-
+
chunk = ChatCompletionChunk(
id=request_id,
choices=[choice],
created=created_time,
model=model,
)
-
+
yield chunk
except requests.exceptions.RequestException as e:
@@ -140,7 +141,7 @@ def _create_non_stream(
proxies=proxies or getattr(self._client, "proxies", None)
)
response.raise_for_status()
-
+
# Process the response
raw_response = response.text.strip('"')
# Format the full response using the client's formatter
@@ -150,26 +151,26 @@ def _create_non_stream(
prompt_tokens = count_tokens(payload.get("query", ""))
completion_tokens = count_tokens(full_response)
total_tokens = prompt_tokens + completion_tokens
-
+
usage = CompletionUsage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=total_tokens
)
-
+
# Create the message object
message = ChatCompletionMessage(
role="assistant",
content=full_response
)
-
+
# Create the choice object
choice = Choice(
index=0,
message=message,
finish_reason="stop"
)
-
+
# Create the completion object
completion = ChatCompletion(
id=request_id,
@@ -178,7 +179,7 @@ def _create_non_stream(
model=model,
usage=usage,
)
-
+
return completion
except Exception as e:
@@ -236,10 +237,10 @@ def __init__(
self.temperature = temperature
self.top_p = top_p
self.system_prompt = system_prompt
-
+
# Initialize LitAgent for user agent generation
agent = LitAgent()
-
+
self.headers = {
'authority': 'netwrck.com',
'accept': '*/*',
@@ -249,10 +250,10 @@ def __init__(
'referer': 'https://netwrck.com/',
'user-agent': agent.random()
}
-
+
self.session = requests.Session()
self.session.headers.update(self.headers)
-
+
# Initialize the chat interface
self.chat = Chat(self)
@@ -300,12 +301,12 @@ def convert_model_name(self, model: str) -> str:
"""
if model in self.AVAILABLE_MODELS:
return model
-
+
# Try to find a matching model
for available_model in self.AVAILABLE_MODELS:
if model.lower() in available_model.lower():
return available_model
-
+
# Default to DeepSeek if no match
print(f"{BOLD}Warning: Model '{model}' not found, using default model 'deepseek/deepseek-r1'{RESET}")
return "deepseek/deepseek-r1"
@@ -342,7 +343,7 @@ def list(inner_self):
],
stream=False
)
-
+
if response and response.choices and response.choices[0].message.content:
status = "✓"
# Truncate response if too long
diff --git a/webscout/Provider/OPENAI/nvidia.py b/webscout/Provider/OPENAI/nvidia.py
index a9a608dd..75a02976 100644
--- a/webscout/Provider/OPENAI/nvidia.py
+++ b/webscout/Provider/OPENAI/nvidia.py
@@ -1,15 +1,19 @@
import json
import time
import uuid
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
from curl_cffi.requests import Session
-from curl_cffi import CurlError
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ count_tokens,
)
try:
@@ -45,10 +49,10 @@ def create(
if top_p is not None:
payload["top_p"] = top_p
payload.update(kwargs)
-
+
request_id = f"chatcmpl-{uuid.uuid4()}"
created_time = int(time.time())
-
+
if stream:
return self._create_stream(request_id, created_time, model, payload, timeout, proxies)
else:
@@ -69,7 +73,7 @@ def _create_stream(
impersonate="chrome120"
)
response.raise_for_status()
-
+
prompt_tokens = count_tokens([msg.get("content", "") for msg in payload.get("messages", [])])
completion_tokens = 0
total_tokens = 0
@@ -101,7 +105,7 @@ def _create_stream(
if content:
completion_tokens += count_tokens(content)
total_tokens = prompt_tokens + completion_tokens
-
+
delta = ChoiceDelta(
content=content,
role=delta_data.get('role'),
@@ -129,7 +133,7 @@ def _create_stream(
yield chunk
except json.JSONDecodeError:
continue
-
+
# Final chunk with finish_reason="stop"
delta = ChoiceDelta(content=None, role=None, tool_calls=None)
choice = Choice(index=0, delta=delta, finish_reason="stop", logprobs=None)
@@ -147,7 +151,7 @@ def _create_stream(
"estimated_cost": None
}
yield chunk
-
+
except Exception as e:
raise IOError(f"Nvidia stream request failed: {e}") from e
@@ -166,10 +170,10 @@ def _create_non_stream(
)
response.raise_for_status()
data = response.json()
-
+
choices_data = data.get('choices', [])
usage_data = data.get('usage', {})
-
+
choices = []
for choice_d in choices_data:
message_d = choice_d.get('message', {})
@@ -183,13 +187,13 @@ def _create_non_stream(
finish_reason=choice_d.get('finish_reason', 'stop')
)
choices.append(choice)
-
+
usage = CompletionUsage(
prompt_tokens=usage_data.get('prompt_tokens', 0),
completion_tokens=usage_data.get('completion_tokens', 0),
total_tokens=usage_data.get('total_tokens', 0)
)
-
+
completion = ChatCompletion(
id=data.get('id', request_id),
choices=choices,
@@ -222,7 +226,7 @@ def get_models(cls, api_key: str = None) -> List[str]:
headers = {}
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
-
+
response = temp_session.get(url, headers=headers, timeout=10)
if response.status_code == 200:
data = response.json()
@@ -246,7 +250,7 @@ def update_available_models(cls, api_key: str = None):
def __init__(self, api_key: str, browser: str = "chrome", timeout: int = 30):
if not api_key:
raise ValueError("API key is required for Nvidia")
-
+
# Update available models from API
self.update_available_models(api_key)
@@ -254,10 +258,10 @@ def __init__(self, api_key: str, browser: str = "chrome", timeout: int = 30):
self.timeout = timeout
self.base_url = "https://integrate.api.nvidia.com/v1/chat/completions"
self.session = Session()
-
+
agent = LitAgent()
fingerprint = agent.generate_fingerprint(browser)
-
+
self.headers = {
"Accept": "application/json",
"Accept-Language": fingerprint["accept_language"],
diff --git a/webscout/Provider/OPENAI/oivscode.py b/webscout/Provider/OPENAI/oivscode.py
index eb2fbc24..17be27f1 100644
--- a/webscout/Provider/OPENAI/oivscode.py
+++ b/webscout/Provider/OPENAI/oivscode.py
@@ -1,18 +1,22 @@
+import json
import random
import secrets
-from regex import F
-import requests
-import json
+import string
import time
import uuid
-import string
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
+
+import requests
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
)
# --- oivscode Client ---
diff --git a/webscout/Provider/OPENAI/sambanova.py b/webscout/Provider/OPENAI/sambanova.py
index edf07937..742cbad1 100644
--- a/webscout/Provider/OPENAI/sambanova.py
+++ b/webscout/Provider/OPENAI/sambanova.py
@@ -6,15 +6,19 @@
import json
import time
import uuid
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
-from curl_cffi.requests import Session
from curl_cffi import CurlError
+from curl_cffi.requests import Session
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
)
try:
@@ -249,35 +253,35 @@ class Sambanova(OpenAICompatibleProvider):
Requires API key from https://cloud.sambanova.ai/
"""
required_auth = True
-
+
AVAILABLE_MODELS = []
@classmethod
- def get_models(cls, api_key: str = None) -> List[str]:
+ def get_models(cls, api_key: Optional[str] = None) -> List[str]:
"""Fetch available models from Sambanova API."""
if not api_key:
raise ValueError("API key is required to fetch models.")
-
+
try:
temp_session = Session()
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {api_key}",
}
-
+
response = temp_session.get(
"https://api.sambanova.ai/v1/models",
headers=headers,
impersonate="chrome120"
)
-
+
if response.status_code == 200:
data = response.json()
if "data" in data and isinstance(data["data"], list):
return [model['id'] for model in data['data'] if 'id' in model]
-
+
return cls.AVAILABLE_MODELS
-
+
except Exception:
return cls.AVAILABLE_MODELS
@@ -289,7 +293,7 @@ def __init__(
):
"""
Initialize the Sambanova OpenAI-compatible client.
-
+
Args:
api_key: Your Sambanova API key (required)
timeout: Request timeout in seconds
@@ -297,13 +301,13 @@ def __init__(
"""
if not api_key:
raise ValueError("API key is required for Sambanova")
-
+
self.api_key = api_key
self.timeout = timeout
self.base_url = "https://api.sambanova.ai/v1/chat/completions"
-
+
self.session = Session()
-
+
# Generate browser fingerprint
if LitAgent:
agent = LitAgent()
@@ -325,17 +329,17 @@ def __init__(
"Content-Type": "application/json",
"Authorization": f"Bearer {api_key}",
}
-
+
self.session.headers.update(self.headers)
-
+
# Update models list dynamically
self.update_available_models(api_key)
-
+
# Initialize chat interface
self.chat = Chat(self)
@classmethod
- def update_available_models(cls, api_key: str = None):
+ def update_available_models(cls, api_key: Optional[str] = None):
"""Update the available models list from Sambanova API."""
if api_key:
models = cls.get_models(api_key)
@@ -353,14 +357,14 @@ def list(inner_self):
if __name__ == "__main__":
# Example usage - requires API key
import os
-
+
api_key = os.environ.get("SAMBANOVA_API_KEY", "")
if not api_key:
print("Set SAMBANOVA_API_KEY environment variable to test")
else:
client = Sambanova(api_key=api_key)
print(f"Available models: {client.models.list()}")
-
+
# Test non-streaming
response = client.chat.completions.create(
model="Meta-Llama-3.1-8B-Instruct",
@@ -368,16 +372,23 @@ def list(inner_self):
max_tokens=100,
stream=False
)
- print(f"Response: {response.choices[0].message.content}")
-
+ if isinstance(response, ChatCompletion):
+ print(f"Response: {response.choices[0].message.content}")
+ else:
+ print(f"Response: {response}")
+
# Test streaming
print("\nStreaming response:")
- for chunk in client.chat.completions.create(
+ stream_resp = client.chat.completions.create(
model="Meta-Llama-3.1-8B-Instruct",
messages=[{"role": "user", "content": "Say hello briefly"}],
max_tokens=100,
stream=True
- ):
- if chunk.choices[0].delta.content:
- print(chunk.choices[0].delta.content, end="", flush=True)
+ )
+ if hasattr(stream_resp, "__iter__") and not isinstance(stream_resp, (str, bytes, ChatCompletion)):
+ for chunk in stream_resp:
+ if chunk.choices[0].delta.content:
+ print(chunk.choices[0].delta.content, end="", flush=True)
+ else:
+ print(stream_resp)
print()
diff --git a/webscout/Provider/OPENAI/sonus.py b/webscout/Provider/OPENAI/sonus.py
index a94f52e2..006d2559 100644
--- a/webscout/Provider/OPENAI/sonus.py
+++ b/webscout/Provider/OPENAI/sonus.py
@@ -1,20 +1,21 @@
+import json
import time
import uuid
+from typing import Any, Dict, Generator, List, Optional, Union
+
import requests
-import json
-from typing import List, Dict, Optional, Union, Generator, Any
from webscout.litagent import LitAgent
from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
ChatCompletion,
ChatCompletionChunk,
- Choice,
ChatCompletionMessage,
+ Choice,
ChoiceDelta,
CompletionUsage,
+ count_tokens,
format_prompt,
- count_tokens
)
# ANSI escape codes for formatting
@@ -47,7 +48,7 @@ def create(
# This creates a conversation in the format: "User: message\nAssistant: response\nUser: message\nAssistant:"
# SonusAI works better with a properly formatted conversation
question = format_prompt(messages, add_special_tokens=True, do_continue=True)
-
+
# Extract reasoning parameter if provided
reasoning = kwargs.get('reasoning', False)
@@ -88,30 +89,30 @@ def _create_stream(
for line in response.iter_lines():
if not line:
continue
-
+
try:
# Decode the line and remove 'data: ' prefix if present
line_text = line.decode('utf-8')
if line_text.startswith('data: '):
line_text = line_text[6:]
-
+
data = json.loads(line_text)
if "content" in data:
content = data["content"]
streaming_text += content
completion_tokens += count_tokens(content)
-
+
# Create a delta object for this chunk
delta = ChoiceDelta(content=content)
choice = Choice(index=0, delta=delta, finish_reason=None)
-
+
chunk = ChatCompletionChunk(
id=request_id,
choices=[choice],
created=created_time,
model=model,
)
-
+
yield chunk
except (json.JSONDecodeError, UnicodeDecodeError):
continue
@@ -119,14 +120,14 @@ def _create_stream(
# Final chunk with finish_reason
delta = ChoiceDelta(content=None)
choice = Choice(index=0, delta=delta, finish_reason="stop")
-
+
chunk = ChatCompletionChunk(
id=request_id,
choices=[choice],
created=created_time,
model=model,
)
-
+
yield chunk
except requests.exceptions.RequestException as e:
@@ -163,26 +164,26 @@ def _create_non_stream(
prompt_tokens = count_tokens(files.get('message', ['',''])[1])
completion_tokens = count_tokens(full_response)
total_tokens = prompt_tokens + completion_tokens
-
+
usage = CompletionUsage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=total_tokens
)
-
+
# Create the message object
message = ChatCompletionMessage(
role="assistant",
content=full_response
)
-
+
# Create the choice object
choice = Choice(
index=0,
message=message,
finish_reason="stop"
)
-
+
# Create the completion object
completion = ChatCompletion(
id=request_id,
@@ -191,7 +192,7 @@ def _create_non_stream(
model=model,
usage=usage,
)
-
+
return completion
except Exception as e:
@@ -233,7 +234,7 @@ def __init__(
"""
self.timeout = timeout
self.url = "https://chat.sonus.ai/chat.php"
-
+
# Headers for the request
agent = LitAgent()
self.headers = {
@@ -243,10 +244,10 @@ def __init__(
'Referer': 'https://chat.sonus.ai/',
'User-Agent': agent.random()
}
-
+
self.session = requests.Session()
self.session.headers.update(self.headers)
-
+
# Initialize the chat interface
self.chat = Chat(self)
@@ -256,12 +257,12 @@ def convert_model_name(self, model: str) -> str:
"""
if model in self.AVAILABLE_MODELS:
return model
-
+
# Try to find a matching model
for available_model in self.AVAILABLE_MODELS:
if model.lower() in available_model.lower():
return available_model
-
+
# Default to pro if no match
print(f"{BOLD}Warning: Model '{model}' not found, using default model 'pro'{RESET}")
return "pro"
@@ -294,8 +295,8 @@ def list(inner_self):
],
stream=False
)
-
- if response and response.choices and response.choices[0].message.content:
+
+ if isinstance(response, ChatCompletion) and response.choices and response.choices[0].message.content:
status = "✓"
# Truncate response if too long
display_text = response.choices[0].message.content.strip()
diff --git a/webscout/Provider/OPENAI/textpollinations.py b/webscout/Provider/OPENAI/textpollinations.py
index c0a77abd..364181e4 100644
--- a/webscout/Provider/OPENAI/textpollinations.py
+++ b/webscout/Provider/OPENAI/textpollinations.py
@@ -3,18 +3,26 @@
https://text.pollinations.ai/openai
"""
+import json
import time
import uuid
+from typing import Any, Dict, Generator, List, Optional, Union
+
import requests
-import json
-from typing import List, Dict, Optional, Union, Generator, Any
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.litagent import LitAgent
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, ToolCall, ToolFunction, count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ ToolCall,
+ ToolFunction,
+ count_tokens,
)
-from webscout.litagent import LitAgent
BOLD = "\033[1m"
RED = "\033[91m"
@@ -254,7 +262,7 @@ class TextPollinations(OpenAICompatibleProvider):
Provides free access to various models including GPT variants and open-source models.
"""
required_auth = False
-
+
AVAILABLE_MODELS = ["openai", "mistral", "p1", "unity"]
@classmethod
@@ -273,7 +281,7 @@ def get_models(cls, api_key: str = None) -> List[str]:
data = response.json()
if isinstance(data, list):
return [model.get("name") for model in data if isinstance(model, dict) and "name" in model]
-
+
return cls.AVAILABLE_MODELS
except Exception:
@@ -339,9 +347,9 @@ def list(inner_self):
response = client.chat.completions.create(
model=model_to_use,
messages=[{"role": "user", "content": "Hello!"}]
- )
+ )
print(response.choices[0].message.content)
except Exception as e:
print(f"Error testing model: {e}")
else:
- print("No models available.")
\ No newline at end of file
+ print("No models available.")
diff --git a/webscout/Provider/OPENAI/toolbaz.py b/webscout/Provider/OPENAI/toolbaz.py
index 03a878c3..a9a8366c 100644
--- a/webscout/Provider/OPENAI/toolbaz.py
+++ b/webscout/Provider/OPENAI/toolbaz.py
@@ -1,25 +1,25 @@
-import time
-import uuid
import base64
import json
import random
-import string
import re
-import cloudscraper
+import string
+import time
+import uuid
from datetime import datetime
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
+
+import cloudscraper
from webscout.litagent import LitAgent
from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
ChatCompletion,
ChatCompletionChunk,
- Choice,
ChatCompletionMessage,
+ Choice,
ChoiceDelta,
CompletionUsage,
format_prompt,
- get_system_prompt
)
# ANSI escape codes for formatting
@@ -50,7 +50,7 @@ def create(
"""
# Format the messages using the format_prompt utility
formatted_prompt = format_prompt(messages, add_special_tokens=True, do_continue=True)
-
+
# Get authentication token
auth = self._client.get_auth()
if not auth:
@@ -97,38 +97,38 @@ def _handle_streaming_response(
buffer = ""
tag_start = "[model:"
streaming_text = ""
-
+
for chunk in resp.iter_content(chunk_size=1):
if chunk:
text = chunk.decode(errors="ignore")
buffer += text
-
+
# Remove all complete [model: ...] tags in buffer
while True:
match = re.search(r"\[model:.*?\]", buffer)
if not match:
break
buffer = buffer[:match.start()] + buffer[match.end():]
-
+
# Only yield up to the last possible start of a tag
last_tag = buffer.rfind(tag_start)
if last_tag == -1 or last_tag + len(tag_start) > len(buffer):
if buffer:
streaming_text += buffer
-
+
# Create the delta object
delta = ChoiceDelta(
content=buffer,
role="assistant"
)
-
+
# Create the choice object
choice = Choice(
index=0,
delta=delta,
finish_reason=None
)
-
+
# Create the chunk object
chunk = ChatCompletionChunk(
id=request_id,
@@ -136,26 +136,26 @@ def _handle_streaming_response(
created=created_time,
model=model
)
-
+
yield chunk
buffer = ""
else:
if buffer[:last_tag]:
streaming_text += buffer[:last_tag]
-
+
# Create the delta object
delta = ChoiceDelta(
content=buffer[:last_tag],
role="assistant"
)
-
+
# Create the choice object
choice = Choice(
index=0,
delta=delta,
finish_reason=None
)
-
+
# Create the chunk object
chunk = ChatCompletionChunk(
id=request_id,
@@ -163,10 +163,10 @@ def _handle_streaming_response(
created=created_time,
model=model
)
-
+
yield chunk
buffer = buffer[last_tag:]
-
+
# Remove any remaining [model: ...] tag in the buffer
buffer = re.sub(r"\[model:.*?\]", "", buffer)
if buffer:
@@ -175,14 +175,14 @@ def _handle_streaming_response(
content=buffer,
role="assistant"
)
-
+
# Create the choice object
choice = Choice(
index=0,
delta=delta,
finish_reason="stop"
)
-
+
# Create the chunk object
chunk = ChatCompletionChunk(
id=request_id,
@@ -190,30 +190,30 @@ def _handle_streaming_response(
created=created_time,
model=model
)
-
+
yield chunk
-
+
# Final chunk with finish_reason
delta = ChoiceDelta(
content=None,
role=None
)
-
+
choice = Choice(
index=0,
delta=delta,
finish_reason="stop"
)
-
+
chunk = ChatCompletionChunk(
id=request_id,
choices=[choice],
created=created_time,
model=model
)
-
+
yield chunk
-
+
except Exception as e:
print(f"{RED}Error during Toolbaz streaming request: {e}{RESET}")
raise IOError(f"Toolbaz streaming request failed: {e}") from e
@@ -246,17 +246,17 @@ def _handle_non_streaming_response(
role="assistant",
content=text
)
-
+
# Create the choice object
choice = Choice(
index=0,
message=message,
finish_reason="stop"
)
-
+
# Usage data is not provided by this API in a standard way, set to 0
usage = CompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
-
+
# Create the completion object
completion = ChatCompletion(
id=request_id,
@@ -265,9 +265,9 @@ def _handle_non_streaming_response(
model=model,
usage=usage
)
-
+
return completion
-
+
except Exception as e:
print(f"{RED}Error during Toolbaz non-stream request: {e}{RESET}")
raise IOError(f"Toolbaz request failed: {e}") from e
@@ -305,7 +305,7 @@ class Toolbaz(OpenAICompatibleProvider):
"grok-4-fast",
"grok-4.1-fast",
-
+
"toolbaz-v4.5-fast",
"toolbaz_v4",
"toolbaz_v3.5_pro",
@@ -316,7 +316,7 @@ class Toolbaz(OpenAICompatibleProvider):
"Llama-4-Maverick",
"Llama-3.3-70B",
-
+
"mixtral_8x22b",
"L3-70B-Euryale-v2.1",
"midnight-rose",
@@ -340,10 +340,10 @@ def __init__(
"""
self.timeout = timeout
self.proxies = proxies
-
+
# Initialize session with cloudscraper
self.session = cloudscraper.create_scraper()
-
+
# Set up headers
self.session.headers.update({
**LitAgent().generate_fingerprint(browser=browser),
diff --git a/webscout/Provider/OPENAI/typefully.py b/webscout/Provider/OPENAI/typefully.py
index dfe45b99..37156586 100644
--- a/webscout/Provider/OPENAI/typefully.py
+++ b/webscout/Provider/OPENAI/typefully.py
@@ -1,30 +1,30 @@
+import json
import time
import uuid
-import json
-import re
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
+
+from curl_cffi import CurlError
+
+# Import curl_cffi for better request handling
+from curl_cffi.requests import Session
+
+# Import LitAgent for browser fingerprinting
+from webscout.litagent import LitAgent
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk,
ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
Choice,
ChoiceDelta,
- ChatCompletionMessage,
CompletionUsage,
+ count_tokens, # Import format_prompt, get_system_prompt and count_tokens
format_prompt,
get_system_prompt,
- count_tokens, # Import format_prompt, get_system_prompt and count_tokens
)
-# Import LitAgent for browser fingerprinting
-from webscout.litagent import LitAgent
-
-# Import curl_cffi for better request handling
-from curl_cffi.requests import Session
-from curl_cffi import CurlError
-
# ANSI escape codes for formatting
BOLD = "\033[1m"
RED = "\033[91m"
@@ -324,8 +324,6 @@ def __init__(
@staticmethod
def _typefully_extractor(chunk) -> str:
"""Extracts content from Typefully AI SSE format."""
- import re
- import json
# Handle parsed JSON objects (when to_json=True)
if isinstance(chunk, dict):
@@ -415,4 +413,7 @@ def list(self):
max_tokens=150,
)
- print(f"{BOLD}Response:{RESET} {response.choices[0].message.content}")
+ if isinstance(response, ChatCompletion):
+ print(f"{BOLD}Response:{RESET} {response.choices[0].message.content}")
+ else:
+ print(f"{BOLD}Response:{RESET} {response}")
diff --git a/webscout/Provider/OPENAI/typliai.py b/webscout/Provider/OPENAI/typliai.py
index 9ea169b3..e55180e0 100644
--- a/webscout/Provider/OPENAI/typliai.py
+++ b/webscout/Provider/OPENAI/typliai.py
@@ -2,22 +2,25 @@
TypliAI OpenAI-compatible provider.
"""
-import json
-import time
-import uuid
import random
import string
-from typing import List, Dict, Optional, Union, Generator, Any
+import time
+import uuid
+from typing import Any, Dict, Generator, List, Optional, Union
from curl_cffi.requests import Session
-from curl_cffi import CurlError
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.AIutel import sanitize_stream
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, format_prompt
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ format_prompt,
)
-from webscout.AIutel import sanitize_stream
try:
from webscout.litagent import LitAgent
@@ -107,10 +110,10 @@ def _create_stream(
full_response = ""
completion_tokens = 0
-
+
# Use chunks from iter_content
data_generator = response.iter_content(chunk_size=None)
-
+
processed_stream = sanitize_stream(
data=data_generator,
intro_value="data: ",
@@ -123,7 +126,7 @@ def _create_stream(
if content and isinstance(content, str):
full_response += content
completion_tokens += 1
-
+
delta = ChoiceDelta(
content=content,
role="assistant" if completion_tokens == 1 else None
@@ -171,7 +174,7 @@ def _create_non_stream(
for chunk in self._create_stream(request_id, created_time, model, prompt, timeout, proxies):
if chunk.choices[0].delta.content:
full_content += chunk.choices[0].delta.content
-
+
message = ChatCompletionMessage(
role="assistant",
content=full_content
@@ -206,12 +209,12 @@ class TypliAI(OpenAICompatibleProvider):
OpenAI-compatible client for TypliAI.
"""
required_auth = False
-
+
AVAILABLE_MODELS = [
- "openai/gpt-4.1-mini",
+ "openai/gpt-4.1-mini",
"openai/gpt-4.1",
- "openai/gpt-5-mini",
- "openai/gpt-5.2",
+ "openai/gpt-5-mini",
+ "openai/gpt-5.2",
"openai/gpt-5.2-pro",
"google/gemini-2.5-flash",
"anthropic/claude-haiku-4-5",
@@ -222,17 +225,17 @@ class TypliAI(OpenAICompatibleProvider):
def __init__(
self,
timeout: int = 60,
- proxies: dict = None,
+ proxies: Optional[dict] = None,
browser: str = "chrome"
):
self.timeout = timeout
self.proxies = proxies or {}
self.api_endpoint = "https://typli.ai/api/generators/chat"
-
+
self.session = Session()
self.agent = LitAgent() if LitAgent else None
user_agent = self.agent.random() if self.agent else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
-
+
self.headers = {
'accept': '/',
'accept-language': 'en-US,en;q=0.9,en-IN;q=0.8',
@@ -250,11 +253,11 @@ def __init__(
'sec-gpc': '1',
'user-agent': user_agent,
}
-
+
self.session.headers.update(self.headers)
if proxies:
self.session.proxies = proxies
-
+
self.chat = Chat(self)
@property
@@ -268,7 +271,7 @@ def list(inner_self):
if __name__ == "__main__":
client = TypliAI()
print(f"Available models: {client.models.list()}")
-
+
# Test non-streaming
print("\n=== Testing Non-Streaming ===")
response = client.chat.completions.create(
@@ -276,15 +279,22 @@ def list(inner_self):
messages=[{"role": "user", "content": "Hello! How are you?"}],
stream=False
)
- print(f"Response: {response.choices[0].message.content}")
-
+ if isinstance(response, ChatCompletion):
+ print(f"Response: {response.choices[0].message.content}")
+ else:
+ print(f"Response: {response}")
+
# Test streaming
print("\n=== Testing Streaming ===")
- for chunk in client.chat.completions.create(
+ stream_resp = client.chat.completions.create(
model="openai/gpt-4.1-mini",
messages=[{"role": "user", "content": "Tell me a joke"}],
stream=True
- ):
- if chunk.choices[0].delta.content:
- print(chunk.choices[0].delta.content, end="", flush=True)
+ )
+ if hasattr(stream_resp, "__iter__") and not isinstance(stream_resp, (str, bytes, ChatCompletion)):
+ for chunk in stream_resp:
+ if chunk.choices[0].delta.content:
+ print(chunk.choices[0].delta.content, end="", flush=True)
+ else:
+ print(stream_resp)
print()
diff --git a/webscout/Provider/OPENAI/utils.py b/webscout/Provider/OPENAI/utils.py
index 1e760fdb..25f8e793 100644
--- a/webscout/Provider/OPENAI/utils.py
+++ b/webscout/Provider/OPENAI/utils.py
@@ -1,10 +1,9 @@
-from typing import List, Dict, Optional, Any
-from enum import Enum
import time
import uuid
-from webscout.Provider.OPENAI.pydantic_imports import (
- BaseModel, Field, StrictStr, StrictInt
-)
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+from webscout.Provider.OPENAI.pydantic_imports import BaseModel, Field, StrictInt, StrictStr
# --- OpenAI Response Structure Mimics ---
# Moved here for reusability across different OpenAI-compatible providers
@@ -297,7 +296,7 @@ def count_tokens(text_or_messages: Any) -> int:
Returns:
int: Number of tokens.
"""
- import tiktoken # type: ignore
+ import tiktoken # type: ignore
if isinstance(text_or_messages, str):
enc = tiktoken.encoding_for_model("gpt-4o")
return len(enc.encode(text_or_messages))
diff --git a/webscout/Provider/OPENAI/venice.py b/webscout/Provider/OPENAI/venice.py
index 87e3d4b8..7c0d2d2a 100644
--- a/webscout/Provider/OPENAI/venice.py
+++ b/webscout/Provider/OPENAI/venice.py
@@ -1,16 +1,21 @@
-import time
-import uuid
import json
import random
-from typing import List, Dict, Optional, Union, Generator, Any
-from curl_cffi import CurlError
+import time
+import uuid
+from typing import Any, Dict, Generator, List, Optional, Union
+
from curl_cffi.requests import Session
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ count_tokens,
)
# Attempt to import LitAgent, fallback if not available
@@ -45,7 +50,7 @@ def create(
# Extract system message if present for systemPrompt parameter
system_prompt = self._client.system_prompt
filtered_messages = []
-
+
for msg in messages:
if msg["role"] == "system":
system_prompt = msg["content"]
@@ -322,7 +327,7 @@ class Venice(OpenAICompatibleProvider):
messages=[{"role": "user", "content": "Hello!"}]
)
"""
- required_auth = False
+ required_auth = False
AVAILABLE_MODELS = [
"mistral-31-24b",
"dolphin-3.0-mistral-24b",
diff --git a/webscout/Provider/OPENAI/wisecat.py b/webscout/Provider/OPENAI/wisecat.py
index f9304e1b..763bf0f5 100644
--- a/webscout/Provider/OPENAI/wisecat.py
+++ b/webscout/Provider/OPENAI/wisecat.py
@@ -1,16 +1,20 @@
+import re
import time
import uuid
-import re
-import json
-from typing import List, Dict, Optional, Union, Generator, Any
-from curl_cffi import CurlError
+from typing import Any, Dict, Generator, List, Optional, Union
+
from curl_cffi.requests import Session
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ count_tokens,
)
# Attempt to import LitAgent, fallback if not available
@@ -334,11 +338,11 @@ def format_text(self, text: str) -> str:
# Handle unicode escaping and quote unescaping
text = text.encode().decode('unicode_escape')
text = text.replace('\\\\', '\\').replace('\\"', '"')
-
+
# Remove timing information
text = re.sub(r'\(\d+\.?\d*s\)', '', text)
text = re.sub(r'\(\d+\.?\d*ms\)', '', text)
-
+
return text
except Exception as e:
# If any error occurs, return the original text
diff --git a/webscout/Provider/OPENAI/writecream.py b/webscout/Provider/OPENAI/writecream.py
index 075b5d05..dd7351e8 100644
--- a/webscout/Provider/OPENAI/writecream.py
+++ b/webscout/Provider/OPENAI/writecream.py
@@ -1,14 +1,20 @@
+import json
import time
import uuid
+from typing import Any, Dict, Generator, List, Optional, Union
+
import requests
-import json
-from typing import List, Dict, Optional, Union, Generator, Any
# Import base classes and utility structures
-from .base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from .base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from .utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ count_tokens,
)
# Attempt to import LitAgent, fallback if not available
@@ -24,8 +30,8 @@ def __init__(self, client: 'Writecream'):
def create(
self,
*,
- model: str = None, # Not used by Writecream, for compatibility
- messages: List[Dict[str, str]],
+ model: str, # Not used by Writecream, for compatibility
+ messages: List[Dict[str, Any]],
max_tokens: Optional[int] = None, # Not used by Writecream
stream: bool = False,
temperature: Optional[float] = None, # Not used by Writecream
@@ -158,9 +164,13 @@ def list(inner_self):
if __name__ == "__main__":
client = Writecream()
response = client.chat.completions.create(
+ model="writecream",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "What is the capital of France?"}
]
)
- print(response.choices[0].message.content)
+ if isinstance(response, ChatCompletion):
+ print(response.choices[0].message.content)
+ else:
+ print(response)
diff --git a/webscout/Provider/OPENAI/x0gpt.py b/webscout/Provider/OPENAI/x0gpt.py
index 657a4448..57c35c34 100644
--- a/webscout/Provider/OPENAI/x0gpt.py
+++ b/webscout/Provider/OPENAI/x0gpt.py
@@ -1,17 +1,21 @@
+import re
import time
import uuid
-import re
-import json
-from typing import List, Dict, Optional, Union, Generator, Any
-from curl_cffi import CurlError
-from curl_cffi.requests import Session
+from typing import Any, Dict, Generator, List, Optional, Union
+
from curl_cffi.const import CurlHttpVersion
+from curl_cffi.requests import Session
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ count_tokens,
)
# Attempt to import LitAgent, fallback if not available
@@ -388,4 +392,4 @@ def list(inner_self):
{"role": "user", "content": "Hello! How are you today?"}
]
)
- print(response.choices[0].message.content)
\ No newline at end of file
+ print(response.choices[0].message.content)
diff --git a/webscout/Provider/OPENAI/yep.py b/webscout/Provider/OPENAI/yep.py
index b359688d..04b65ec9 100644
--- a/webscout/Provider/OPENAI/yep.py
+++ b/webscout/Provider/OPENAI/yep.py
@@ -1,14 +1,20 @@
+import json
import time
import uuid
+from typing import Any, Dict, Generator, List, Optional, Union
+
import cloudscraper # Import cloudscraper
-import json
-from typing import List, Dict, Optional, Union, Generator, Any
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage, get_system_prompt, count_tokens # Import count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage, # Import count_tokens
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
+ count_tokens,
)
# Attempt to import LitAgent, fallback if not available
@@ -194,7 +200,7 @@ def _create_stream(
}
yield chunk
- except cloudscraper.exceptions.CloudflareChallengeError as e:
+ except cloudscraper.exceptions.CloudflareChallengeError:
pass
def _create_non_stream(
@@ -391,4 +397,4 @@ def convert_model_name(self, model: str) -> str:
print() # Add a newline at the end
except Exception as e:
- print(f"Streaming Test Failed: {e}")
\ No newline at end of file
+ print(f"Streaming Test Failed: {e}")
diff --git a/webscout/Provider/OPENAI/zenmux.py b/webscout/Provider/OPENAI/zenmux.py
index a3f54625..d3c731cc 100644
--- a/webscout/Provider/OPENAI/zenmux.py
+++ b/webscout/Provider/OPENAI/zenmux.py
@@ -1,16 +1,17 @@
-import requests
import json
import time
import uuid
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
+
+import requests
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk,
ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
Choice,
ChoiceDelta,
- ChatCompletionMessage,
CompletionUsage,
)
@@ -238,7 +239,7 @@ class Zenmux(OpenAICompatibleProvider):
"z-ai/glm-4.6v-flash",
]
- def __init__(self, browser: str = "chrome", api_key: str = None):
+ def __init__(self, browser: str = "chrome", api_key: Optional[str] = None):
self.timeout = None
self.base_url = "https://zenmux.ai/api/v1/chat/completions"
self.session = requests.Session()
@@ -281,12 +282,10 @@ def get_models(cls, api_key: Optional[str] = None):
}
try:
from curl_cffi.requests import Session as CurlSession
- from curl_cffi import CurlError
curl_available = True
except Exception:
CurlSession = None
- CurlError = Exception
curl_available = False
try:
from webscout.litagent import LitAgent
@@ -308,7 +307,7 @@ def get_models(cls, api_key: Optional[str] = None):
pass
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
- if curl_available:
+ if curl_available and CurlSession is not None:
session = CurlSession()
response = session.get(
"https://zenmux.ai/api/v1/models",
diff --git a/webscout/Provider/Openai.py b/webscout/Provider/Openai.py
index 4e3a958f..96d57472 100644
--- a/webscout/Provider/Openai.py
+++ b/webscout/Provider/Openai.py
@@ -1,17 +1,15 @@
import json
-import os
-from typing import Any, Dict, Optional, Generator, Union, List
+from typing import Any, Dict, Generator, Optional, Union
-from curl_cffi.requests import Session
from curl_cffi import CurlError
+from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
from webscout.litagent import LitAgent
+
class OPENAI(Provider):
"""
A class to interact with the OpenAI API with LitAgent user-agent.
@@ -19,12 +17,12 @@ class OPENAI(Provider):
required_auth = True
@classmethod
- def get_models(cls, api_key: str = None):
+ def get_models(cls, api_key: Optional[str] = None):
"""Fetch available models from OpenAI API.
-
+
Args:
api_key (str, optional): OpenAI API key
-
+
Returns:
list: List of available model IDs
"""
@@ -36,21 +34,21 @@ def get_models(cls, api_key: str = None):
headers = {
"Authorization": f"Bearer {api_key}",
}
-
+
response = temp_session.get(
"https://api.openai.com/v1/models",
headers=headers,
impersonate="chrome110"
)
-
+
if response.status_code != 200:
raise Exception(f"API request failed with status {response.status_code}: {response.text}")
-
+
data = response.json()
if "data" in data and isinstance(data["data"], list):
return [model["id"] for model in data["data"] if "id" in model]
raise Exception("Invalid response format from API")
-
+
except (CurlError, Exception) as e:
raise Exception(f"Failed to fetch models: {str(e)}")
@@ -72,12 +70,12 @@ def __init__(
top_p: float = 1,
model: str = "gpt-3.5-turbo",
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
base_url: str = "https://api.openai.com/v1/chat/completions",
system_prompt: str = "You are a helpful assistant.",
browser: str = "chrome"
@@ -167,9 +165,10 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Union[Dict[str, Any], Generator]:
+ **kwargs: Any,
+ ) -> Response:
conversation_prompt = self.conversation.gen_complete_prompt(prompt)
if optimizer:
if optimizer in self.__available_optimizers:
@@ -279,8 +278,9 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
+ **kwargs: Any,
) -> Union[str, Generator[str, None, None]]:
def for_stream_chat():
gen = self.ask(
@@ -301,4 +301,4 @@ def for_non_stream_chat():
def get_message(self, response: dict) -> str:
assert isinstance(response, dict), "Response should be of dict data-type only"
- return response["text"]
\ No newline at end of file
+ return response["text"]
diff --git a/webscout/Provider/PI.py b/webscout/Provider/PI.py
index ec7b2768..47e91bce 100644
--- a/webscout/Provider/PI.py
+++ b/webscout/Provider/PI.py
@@ -1,16 +1,21 @@
-from uuid import uuid4
-from curl_cffi.requests import Session
-from curl_cffi import CurlError
-import json
import re
import threading
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation, sanitize_stream # Import sanitize_stream
-from webscout.AIutel import AwesomePrompts
-from webscout.AIbase import Provider
-from typing import Dict, Union, Any, Optional
-from webscout.litagent import LitAgent
+from typing import Any, Dict, Generator, Optional, Union
+from uuid import uuid4
+
+from curl_cffi import CurlError
+from curl_cffi.requests import Session
+
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import ( # Import sanitize_stream
+ AwesomePrompts,
+ Conversation,
+ Optimizers,
+ sanitize_stream,
+)
+from webscout.litagent import LitAgent
+
class PiAI(Provider):
"""
@@ -39,12 +44,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 2048, # Note: max_tokens is not used by this API
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
voice: bool = False,
voice_name: str = "voice3",
output_file: str = "PiAI.mp3",
@@ -168,12 +173,13 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- voice: bool = None,
- voice_name: str = None,
- output_file: str = None
- ) -> Union[dict, str, Any]:
+ voice: Optional[bool] = None,
+ voice_name: Optional[str] = None,
+ output_file: Optional[str] = None,
+ **kwargs: Any,
+ ) -> Response:
"""
Interact with Pi.ai by sending a prompt and receiving a response.
Now supports raw streaming and non-streaming output, matching the pattern in other providers.
@@ -292,13 +298,14 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- voice: bool = None,
- voice_name: str = None,
- output_file: str = None,
+ voice: Optional[bool] = None,
+ voice_name: Optional[str] = None,
+ output_file: Optional[str] = None,
raw: bool = False, # Added raw parameter
- ) -> Union[str, Any]:
+ **kwargs: Any,
+ ) -> Union[str, Generator[str, None, None]]:
"""
Generates a response based on the provided prompt.
@@ -353,10 +360,11 @@ def stream_generator():
else:
return self.get_message(response_data)
- def get_message(self, response: dict) -> str:
+ def get_message(self, response: Response) -> str:
"""Retrieves message only from response"""
- assert isinstance(response, dict), "Response should be of dict data-type only"
- return response["text"]
+ if not isinstance(response, dict):
+ return str(response)
+ return response.get("text", "")
def download_audio_threaded(self, voice_name: str, second_sid: str, output_file: str) -> None:
"""Downloads audio in a separate thread."""
@@ -396,9 +404,11 @@ def download_audio_threaded(self, voice_name: str, second_sid: str, output_file:
ai = PiAI(timeout=60)
print("[bold blue]Testing Chat (Stream):[/bold blue]")
response = ai.chat("hi", stream=True, raw=False)
- full_response = ""
- for chunk in response:
- print(chunk, end="", flush=True)
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)):
+ for chunk in response:
+ print(chunk, end="", flush=True)
+ else:
+ print(response)
except exceptions.FailedToGenerateResponseError as e:
print(f"\n[bold red]API Error:[/bold red] {e}")
except Exception as e:
diff --git a/webscout/Provider/QwenLM.py b/webscout/Provider/QwenLM.py
index 9f48ff9e..7d9c5225 100644
--- a/webscout/Provider/QwenLM.py
+++ b/webscout/Provider/QwenLM.py
@@ -5,7 +5,7 @@
from curl_cffi import Session
from webscout import exceptions
-from webscout.AIbase import Provider
+from webscout.AIbase import Provider, Response
from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
@@ -143,7 +143,8 @@ def ask(
raw: bool = False,
optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Union[Dict[str, Any], Generator[Any, None, None]]:
+ **kwargs: Any,
+ ) -> Response:
"""Chat with AI."""
conversation_prompt = self.conversation.gen_complete_prompt(prompt)
@@ -307,9 +308,10 @@ def for_non_stream() -> str:
return for_stream() if stream else for_non_stream()
- def get_message(self, response: dict) -> str:
+ def get_message(self, response: Response) -> str:
"""Extracts the message content from a response dict."""
- assert isinstance(response, dict), "Response should be a dict"
+ if not isinstance(response, dict):
+ return str(response)
return response.get("text", "")
if __name__ == "__main__":
diff --git a/webscout/Provider/STT/__init__.py b/webscout/Provider/STT/__init__.py
index c509c78a..a6284e69 100644
--- a/webscout/Provider/STT/__init__.py
+++ b/webscout/Provider/STT/__init__.py
@@ -3,12 +3,12 @@
# Base classes
from webscout.Provider.STT.base import (
- STTCompatibleProvider,
- BaseSTTTranscriptions,
BaseSTTAudio,
BaseSTTChat,
- TranscriptionResponse,
+ BaseSTTTranscriptions,
+ STTCompatibleProvider,
STTModels,
+ TranscriptionResponse,
)
# Provider implementations
diff --git a/webscout/Provider/STT/base.py b/webscout/Provider/STT/base.py
index 5c245e42..8662d18d 100644
--- a/webscout/Provider/STT/base.py
+++ b/webscout/Provider/STT/base.py
@@ -8,41 +8,46 @@
import json
import time
from abc import ABC, abstractmethod
-from typing import Any, Dict, Generator, List, Optional, Union, BinaryIO
from pathlib import Path
+from typing import Any, BinaryIO, Dict, Generator, List, Optional, Union
# Import OpenAI response types from the main OPENAI module
try:
from webscout.Provider.OPENAI.pydantic_imports import (
- ChatCompletion, ChatCompletionChunk, Choice, ChoiceDelta,
- Message, Usage, count_tokens
+ ChatCompletion,
+ ChatCompletionChunk,
+ Choice,
+ ChoiceDelta,
+ Message,
+ Usage,
+ count_tokens,
)
except ImportError:
# Fallback if pydantic_imports is not available
from dataclasses import dataclass
-
+
@dataclass
class Usage:
prompt_tokens: int = 0
completion_tokens: int = 0
total_tokens: int = 0
-
+
@dataclass
class Message:
role: str
content: str
-
+
@dataclass
class Choice:
index: int
message: Message
finish_reason: Optional[str] = None
-
+
@dataclass
class ChoiceDelta:
content: Optional[str] = None
role: Optional[str] = None
-
+
@dataclass
class ChatCompletionChunk:
id: str
@@ -50,7 +55,7 @@ class ChatCompletionChunk:
created: int
model: str
object: str = "chat.completion.chunk"
-
+
@dataclass
class ChatCompletion:
id: str
@@ -59,43 +64,43 @@ class ChatCompletion:
model: str
usage: Usage
object: str = "chat.completion"
-
+
def count_tokens(text: str) -> int:
return len(text.split())
class TranscriptionResponse:
"""Response object that mimics OpenAI's transcription response."""
-
+
def __init__(self, data: Dict[str, Any], response_format: str = "json"):
self._data = data
self._response_format = response_format
-
+
@property
def text(self) -> str:
"""Get the transcribed text."""
return self._data.get("text", "")
-
+
@property
def language(self) -> Optional[str]:
"""Get the detected language."""
return self._data.get("language")
-
+
@property
def duration(self) -> Optional[float]:
"""Get the audio duration."""
return self._data.get("duration")
-
+
@property
def segments(self) -> Optional[list]:
"""Get the segments with timestamps."""
return self._data.get("segments")
-
+
@property
def words(self) -> Optional[list]:
"""Get the words with timestamps."""
return self._data.get("words")
-
+
def __str__(self) -> str:
"""Return string representation based on response format."""
if self._response_format == "text":
@@ -106,42 +111,42 @@ def __str__(self) -> str:
return self._to_vtt()
else: # json or verbose_json
return json.dumps(self._data, indent=2)
-
+
def _to_srt(self) -> str:
"""Convert to SRT subtitle format."""
if not self.segments:
return ""
-
+
srt_content = []
for i, segment in enumerate(self.segments, 1):
start_time = self._format_time_srt(segment.get("start", 0))
end_time = self._format_time_srt(segment.get("end", 0))
text = segment.get("text", "").strip()
-
+
srt_content.append(f"{i}")
srt_content.append(f"{start_time} --> {end_time}")
srt_content.append(text)
srt_content.append("")
-
+
return "\n".join(srt_content)
-
+
def _to_vtt(self) -> str:
"""Convert to VTT subtitle format."""
if not self.segments:
return "WEBVTT\n\n"
-
+
vtt_content = ["WEBVTT", ""]
for segment in self.segments:
start_time = self._format_time_vtt(segment.get("start", 0))
end_time = self._format_time_vtt(segment.get("end", 0))
text = segment.get("text", "").strip()
-
+
vtt_content.append(f"{start_time} --> {end_time}")
vtt_content.append(text)
vtt_content.append("")
-
+
return "\n".join(vtt_content)
-
+
def _format_time_srt(self, seconds: float) -> str:
"""Format time for SRT format (HH:MM:SS,mmm)."""
hours = int(seconds // 3600)
@@ -149,7 +154,7 @@ def _format_time_srt(self, seconds: float) -> str:
secs = int(seconds % 60)
millisecs = int((seconds % 1) * 1000)
return f"{hours:02d}:{minutes:02d}:{secs:02d},{millisecs:03d}"
-
+
def _format_time_vtt(self, seconds: float) -> str:
"""Format time for VTT format (HH:MM:SS.mmm)."""
hours = int(seconds // 3600)
@@ -161,10 +166,10 @@ def _format_time_vtt(self, seconds: float) -> str:
class BaseSTTTranscriptions(ABC):
"""Base class for STT transcriptions interface."""
-
+
def __init__(self, client):
self._client = client
-
+
@abstractmethod
def create(
self,
@@ -183,7 +188,7 @@ def create(
) -> Union[TranscriptionResponse, Generator[str, None, None]]:
"""
Create a transcription of the given audio file.
-
+
Args:
model: Model to use for transcription
file: Audio file to transcribe
@@ -196,7 +201,7 @@ def create(
timeout: Request timeout
proxies: Proxy configuration
**kwargs: Additional parameters
-
+
Returns:
TranscriptionResponse or generator of SSE strings if streaming
"""
@@ -205,10 +210,10 @@ def create(
class BaseSTTAudio(ABC):
"""Base class for STT audio interface."""
-
+
def __init__(self, client):
self.transcriptions = self._create_transcriptions(client)
-
+
@abstractmethod
def _create_transcriptions(self, client) -> BaseSTTTranscriptions:
"""Create the transcriptions interface."""
@@ -228,14 +233,14 @@ class STTCompatibleProvider(ABC):
Abstract Base Class for STT providers mimicking the OpenAI structure.
Requires a nested 'audio.transcriptions' structure.
"""
-
+
audio: BaseSTTAudio
-
+
@abstractmethod
def __init__(self, **kwargs: Any):
"""Initialize the STT provider."""
pass
-
+
@property
@abstractmethod
def models(self):
@@ -247,10 +252,10 @@ def models(self):
class STTModels:
"""Models interface for STT providers."""
-
+
def __init__(self, available_models: List[str]):
self._available_models = available_models
-
+
def list(self) -> List[Dict[str, Any]]:
"""List available models."""
return [
@@ -266,7 +271,7 @@ def list(self) -> List[Dict[str, Any]]:
__all__ = [
'TranscriptionResponse',
- 'BaseSTTTranscriptions',
+ 'BaseSTTTranscriptions',
'BaseSTTAudio',
'BaseSTTChat',
'STTCompatibleProvider',
diff --git a/webscout/Provider/STT/elevenlabs.py b/webscout/Provider/STT/elevenlabs.py
index c5e28e52..b0423845 100644
--- a/webscout/Provider/STT/elevenlabs.py
+++ b/webscout/Provider/STT/elevenlabs.py
@@ -5,25 +5,25 @@
speech-to-text transcription service.
"""
-import json
-import time
-import uuid
from pathlib import Path
-from typing import Any, Dict, Generator, List, Optional, Union, BinaryIO
+from typing import Any, BinaryIO, Generator, List, Optional, Union
import requests
-from webscout.litagent import LitAgent
-from webscout import exceptions
+from webscout import exceptions
+from webscout.litagent import LitAgent
from webscout.Provider.STT.base import (
- BaseSTTTranscriptions, BaseSTTAudio, STTCompatibleProvider,
- STTModels, TranscriptionResponse
+ BaseSTTAudio,
+ BaseSTTTranscriptions,
+ STTCompatibleProvider,
+ STTModels,
+ TranscriptionResponse,
)
class ElevenLabsTranscriptions(BaseSTTTranscriptions):
"""ElevenLabs transcriptions interface."""
-
+
def create(
self,
*,
@@ -181,12 +181,12 @@ def _create_stream(
for line in response.iter_lines(decode_unicode=True):
if line:
yield line
-
+
class ElevenLabsAudio(BaseSTTAudio):
"""ElevenLabs audio interface."""
-
+
def _create_transcriptions(self, client) -> ElevenLabsTranscriptions:
return ElevenLabsTranscriptions(client)
@@ -194,7 +194,7 @@ def _create_transcriptions(self, client) -> ElevenLabsTranscriptions:
class ElevenLabsSTT(STTCompatibleProvider):
"""
OpenAI-compatible client for ElevenLabs STT API.
-
+
Usage:
client = ElevenLabsSTT()
audio_file = open("audio.mp3", "rb")
@@ -205,11 +205,11 @@ class ElevenLabsSTT(STTCompatibleProvider):
)
print(transcription.text)
"""
-
+
AVAILABLE_MODELS = [
"scribe_v1",
]
-
+
def __init__(
self,
model_id: str = "scribe_v1",
@@ -226,14 +226,14 @@ def __init__(
self.diarize = diarize
self.timeout = timeout
self.proxies = proxies
-
+
# API configuration
self.api_url = "https://api.elevenlabs.io/v1/speech-to-text"
-
+
# Initialize interfaces
self.audio = ElevenLabsAudio(self)
self._models = STTModels(self.AVAILABLE_MODELS)
-
+
@property
def models(self):
"""Get models interface."""
@@ -262,4 +262,4 @@ def models(self):
stream=True
)
for chunk in stream:
- print(chunk.strip())
\ No newline at end of file
+ print(chunk.strip())
diff --git a/webscout/Provider/Sambanova.py b/webscout/Provider/Sambanova.py
index b7e5b17a..4b5d246a 100644
--- a/webscout/Provider/Sambanova.py
+++ b/webscout/Provider/Sambanova.py
@@ -1,13 +1,15 @@
-from curl_cffi.requests import Session
-from curl_cffi import CurlError
import json
-from typing import Union, Any, Dict, Generator, Optional, List
+from typing import Any, Dict, Generator, List, Optional, Union
+
+from curl_cffi import CurlError
+from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers, Conversation, AwesomePrompts, sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
from webscout.litagent import LitAgent as Lit
+
class Sambanova(Provider):
"""
A class to interact with the Sambanova API.
@@ -28,42 +30,42 @@ def update_available_models(self):
"""Update the available models list from Sambanova API."""
if not self.api_key:
return
-
+
try:
temp_session = Session()
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.api_key}",
}
-
+
response = temp_session.get(
"https://api.sambanova.ai/v1/models",
headers=headers,
impersonate="chrome120"
)
-
+
if response.status_code == 200:
data = response.json()
if "data" in data and isinstance(data["data"], list):
new_models = [model['id'] for model in data['data'] if 'id' in model]
if new_models:
self.AVAILABLE_MODELS = new_models
-
+
except Exception:
pass
def __init__(
self,
- api_key: str = None,
+ api_key: Optional[str] = None,
is_conversation: bool = True,
max_tokens: int = 4096,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
model: str = "Meta-Llama-3.1-8B-Instruct",
system_prompt: str = "You are a helpful AI assistant.",
):
@@ -74,7 +76,7 @@ def __init__(
self.model = model
self.system_prompt = system_prompt
self.timeout = timeout
-
+
# Update models list dynamically if API key is provided
if api_key:
self.update_available_models()
@@ -90,7 +92,7 @@ def __init__(
"Content-Type": "application/json",
"User-Agent": Lit().random(),
}
-
+
# Update curl_cffi session headers and proxies
self.session.headers.update(self.headers)
self.session.proxies = proxies
@@ -120,11 +122,12 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
tools: Optional[List[Dict[str, Any]]] = None,
tool_choice: Optional[Dict[str, Any]] = None,
- ) -> Union[Any, Generator[Any, None, None]]:
+ **kwargs: Any,
+ ) -> Response:
"""Chat with AI using the Sambanova API."""
conversation_prompt = self.conversation.gen_complete_prompt(prompt)
if optimizer:
@@ -156,13 +159,13 @@ def for_stream():
try:
# Use curl_cffi session post with impersonate
response = self.session.post(
- self.base_url,
- json=payload,
- stream=True,
+ self.base_url,
+ json=payload,
+ stream=True,
timeout=self.timeout,
impersonate="chrome120"
)
- response.raise_for_status()
+ response.raise_for_status()
streaming_text = ""
processed_stream = sanitize_stream(
@@ -242,18 +245,18 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
tools: Optional[List[Dict[str, Any]]] = None,
tool_choice: Optional[Dict[str, Any]] = None,
raw: bool = False,
) -> Union[str, Generator[str, None, None]]:
"""Generate response `str`"""
-
+
def for_stream_chat():
# ask() yields dicts or strings when streaming
gen = self.ask(
- prompt, stream=True, raw=raw,
+ prompt, stream=True, raw=raw,
optimizer=optimizer, conversationally=conversationally,
tools=tools, tool_choice=tool_choice
)
@@ -261,14 +264,14 @@ def for_stream_chat():
if raw:
yield response_dict
else:
- yield self.get_message(response_dict)
+ yield self.get_message(response_dict)
def for_non_stream_chat():
# ask() returns dict or str when not streaming
response_data = self.ask(
prompt,
stream=False,
- raw=raw,
+ raw=raw,
optimizer=optimizer,
conversationally=conversationally,
tools=tools,
@@ -276,11 +279,11 @@ def for_non_stream_chat():
)
if raw:
return response_data
- return self.get_message(response_data)
+ return self.get_message(response_data)
return for_stream_chat() if stream else for_non_stream_chat()
- def get_message(self, response: Any) -> str:
+ def get_message(self, response: Response) -> str:
"""
Retrieves a clean message from the provided response.
@@ -297,12 +300,15 @@ def get_message(self, response: Any) -> str:
return response["text"]
elif "tool_calls" in response:
return json.dumps(response["tool_calls"])
- return ""
+ return str(response)
if __name__ == "__main__":
# Ensure curl_cffi is installed
from rich import print
ai = Sambanova(api_key='')
response = ai.chat(input(">>> "), stream=True)
- for chunk in response:
- print(chunk, end="", flush=True)
\ No newline at end of file
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)):
+ for chunk in response:
+ print(chunk, end="", flush=True)
+ else:
+ print(response)
diff --git a/webscout/Provider/TTI/__init__.py b/webscout/Provider/TTI/__init__.py
index 0b763b6e..acbeddce 100644
--- a/webscout/Provider/TTI/__init__.py
+++ b/webscout/Provider/TTI/__init__.py
@@ -3,23 +3,23 @@
# Base classes
from webscout.Provider.TTI.base import (
- TTICompatibleProvider,
BaseImages,
-)
-
-# Utility classes
-from webscout.Provider.TTI.utils import (
- ImageData,
- ImageResponse,
+ TTICompatibleProvider,
)
# Provider implementations
from webscout.Provider.TTI.claudeonline import ClaudeOnlineTTI
from webscout.Provider.TTI.magicstudio import MagicStudioAI
+from webscout.Provider.TTI.miragic import MiragicAI
from webscout.Provider.TTI.pollinations import PollinationsAI
from webscout.Provider.TTI.together import TogetherImage
+
+# Utility classes
+from webscout.Provider.TTI.utils import (
+ ImageData,
+ ImageResponse,
+)
from webscout.Provider.TTI.venice import VeniceAI
-from webscout.Provider.TTI.miragic import MiragicAI
# List of all exported names
__all__ = [
@@ -37,4 +37,4 @@
"VeniceAI",
"MiragicAI",
-]
\ No newline at end of file
+]
diff --git a/webscout/Provider/TTI/base.py b/webscout/Provider/TTI/base.py
index 231dba36..e10bc2b6 100644
--- a/webscout/Provider/TTI/base.py
+++ b/webscout/Provider/TTI/base.py
@@ -1,7 +1,7 @@
from abc import ABC, abstractmethod
from typing import Any, Optional
-from .utils import ImageResponse
+from .utils import ImageResponse
class BaseImages(ABC):
diff --git a/webscout/Provider/TTI/magicstudio.py b/webscout/Provider/TTI/magicstudio.py
index a96a8505..4706e8f4 100644
--- a/webscout/Provider/TTI/magicstudio.py
+++ b/webscout/Provider/TTI/magicstudio.py
@@ -1,16 +1,15 @@
-import requests
import os
-import uuid
-import time
import tempfile
-from typing import Optional, List
-from webscout.Provider.TTI.utils import (
- ImageData,
- ImageResponse
-)
-from webscout.Provider.TTI.base import TTICompatibleProvider, BaseImages
+import time
+import uuid
from io import BytesIO
+from typing import Optional
+
+import requests
+
from webscout.litagent import LitAgent
+from webscout.Provider.TTI.base import BaseImages, TTICompatibleProvider
+from webscout.Provider.TTI.utils import ImageData, ImageResponse
try:
from PIL import Image
diff --git a/webscout/Provider/TTI/miragic.py b/webscout/Provider/TTI/miragic.py
index 50dc48ea..b14b904a 100644
--- a/webscout/Provider/TTI/miragic.py
+++ b/webscout/Provider/TTI/miragic.py
@@ -1,11 +1,13 @@
-import requests
import json
import random
-import time
-from typing import Optional, List, Any, Dict
-from webscout.Provider.TTI.base import TTICompatibleProvider, BaseImages
-from webscout.Provider.TTI.utils import ImageData, ImageResponse
+from typing import Any, Optional
+
+import requests
+
from webscout.litagent import LitAgent
+from webscout.Provider.TTI.base import BaseImages, TTICompatibleProvider
+from webscout.Provider.TTI.utils import ImageData, ImageResponse
+
class Images(BaseImages):
"""Handles image generation requests for the Miragic AI provider."""
@@ -63,19 +65,19 @@ def create(
width, height = int(parts[0]), int(parts[1])
except ValueError:
pass
-
+
width = max(256, min(2048, width))
height = max(256, min(2048, height))
-
+
enhance = kwargs.get("enhance_prompt", False)
safe = kwargs.get("safe_filter", False)
image_url = kwargs.get("image_url")
-
+
images_data = []
-
+
for _ in range(n):
current_seed = seed if seed is not None else random.randint(0, 2**32 - 1)
-
+
payload = {
"data": [
prompt,
@@ -88,26 +90,26 @@ def create(
safe
]
}
-
+
try:
post_url = f"{self._client.api_endpoint}/call/generate_image_via_api_secure"
resp = self._client.session.post(post_url, json=payload, timeout=timeout)
resp.raise_for_status()
-
+
event_id = resp.json().get("event_id")
if not event_id:
raise RuntimeError(f"Failed to obtain event_id: {resp.text}")
-
+
stream_url = f"{self._client.api_endpoint}/call/generate_image_via_api_secure/{event_id}"
image_url_result = None
-
+
with self._client.session.get(stream_url, stream=True, timeout=timeout + 60) as stream_resp:
stream_resp.raise_for_status()
for line in stream_resp.iter_lines():
if not line:
continue
line_text = line.decode('utf-8')
-
+
if line_text.startswith('data: '):
data_str = line_text[6:]
try:
@@ -120,7 +122,7 @@ def create(
continue
except Exception as e:
raise RuntimeError(f"Image generation failed: {e}")
-
+
if image_url_result:
images_data.append(ImageData(url=image_url_result))
else:
@@ -131,15 +133,15 @@ def create(
class MiragicAI(TTICompatibleProvider):
"""
Miragic AI TTI Provider implementation.
-
+
Reverse engineered from the Hugging Face Space:
https://huggingface.co/spaces/Miragic-AI/Miragic-AI-Image-Generator
"""
-
+
required_auth: bool = False
working: bool = True
AVAILABLE_MODELS = ["flux", "turbo", "gptimage"]
-
+
def __init__(self, **kwargs: Any):
"""Initializes the MiragicAI provider with a persistent session."""
self.api_endpoint = "https://miragic-ai-miragic-ai-image-generator.hf.space/gradio_api"
@@ -151,7 +153,7 @@ def __init__(self, **kwargs: Any):
"Content-Type": "application/json"
})
self.images = Images(self)
-
+
@property
def models(self):
"""Returns a list of available models for this provider."""
@@ -162,11 +164,11 @@ def list(inner_self):
if __name__ == "__main__":
from rich import print
-
+
try:
client = MiragicAI()
print(f"Available Models: {client.models.list()}")
-
+
print("Generating sample image...")
response = client.images.create(
prompt="A serene landscape with a lake and mountains, oil painting style",
@@ -174,6 +176,6 @@ def list(inner_self):
size="1024x1024"
)
print(response)
-
+
except Exception as error:
- print(f"Error during execution: {error}")
\ No newline at end of file
+ print(f"Error during execution: {error}")
diff --git a/webscout/Provider/TTI/pollinations.py b/webscout/Provider/TTI/pollinations.py
index 9bffc084..d0895c27 100644
--- a/webscout/Provider/TTI/pollinations.py
+++ b/webscout/Provider/TTI/pollinations.py
@@ -1,19 +1,17 @@
-import requests
-from typing import Optional, List, Dict, Any, Union
-from pathlib import Path
-from requests.exceptions import RequestException
-from webscout.Provider.TTI.utils import (
- ImageData,
- ImageResponse
-)
-from webscout.Provider.TTI.base import TTICompatibleProvider, BaseImages
-from io import BytesIO
+import json
import os
+import random
import tempfile
-from webscout.litagent import LitAgent
import time
-import json
-import random
+from io import BytesIO
+from typing import Optional
+
+import requests
+from requests.exceptions import RequestException
+
+from webscout.litagent import LitAgent
+from webscout.Provider.TTI.base import BaseImages, TTICompatibleProvider
+from webscout.Provider.TTI.utils import ImageData, ImageResponse
try:
from PIL import Image
diff --git a/webscout/Provider/TTI/together.py b/webscout/Provider/TTI/together.py
index 21391620..c9584d5e 100644
--- a/webscout/Provider/TTI/together.py
+++ b/webscout/Provider/TTI/together.py
@@ -1,6 +1,6 @@
import json
import random
-from typing import Dict, Optional, Any, List
+from typing import Dict, Optional
import requests
from requests.adapters import HTTPAdapter
@@ -262,7 +262,7 @@ def get_models(cls, api_key: str = None):
for model in models_data:
if isinstance(model, dict) and model.get("type", "").lower() == "image":
image_models.append(model["id"])
-
+
if image_models:
return sorted(image_models)
else:
@@ -284,9 +284,9 @@ def update_available_models(cls, api_key=None):
def __init__(self, api_key: str = None):
"""
Initialize the TogetherImage client.
-
+
Args:
- api_key (str, optional): Together.xyz API key.
+ api_key (str, optional): Together.xyz API key.
"""
self.api_key = api_key
# Update available models if API key is provided
@@ -331,4 +331,4 @@ def convert_model_name(self, model: str) -> str:
response_format="url",
timeout=120,
)
- print(response)
\ No newline at end of file
+ print(response)
diff --git a/webscout/Provider/TTI/utils.py b/webscout/Provider/TTI/utils.py
index b24d7a74..71494b3e 100644
--- a/webscout/Provider/TTI/utils.py
+++ b/webscout/Provider/TTI/utils.py
@@ -1,7 +1,9 @@
import time
from typing import List, Optional
+
from pydantic import BaseModel, Field
+
class ImageData(BaseModel):
url: Optional[str] = None
b64_json: Optional[str] = None
diff --git a/webscout/Provider/TTI/venice.py b/webscout/Provider/TTI/venice.py
index ce4c5999..28b91fca 100644
--- a/webscout/Provider/TTI/venice.py
+++ b/webscout/Provider/TTI/venice.py
@@ -11,21 +11,20 @@
>>> print(response)
"""
-import requests
-from typing import Optional, List, Dict, Any
-from webscout.Provider.TTI.utils import (
- ImageData,
- ImageResponse
-)
-from webscout.Provider.TTI.base import TTICompatibleProvider, BaseImages
-from io import BytesIO
-import os
-import tempfile
-from webscout.litagent import LitAgent
-import time
import json
+import os
import random
import string
+import tempfile
+import time
+from io import BytesIO
+from typing import Optional
+
+import requests
+
+from webscout.litagent import LitAgent
+from webscout.Provider.TTI.base import BaseImages, TTICompatibleProvider
+from webscout.Provider.TTI.utils import ImageData, ImageResponse
try:
from PIL import Image
@@ -57,7 +56,7 @@ def create(
) -> ImageResponse:
"""
Create images using Venice AI API.
-
+
Args:
model: The model to use for image generation
prompt: Text description of the image to generate
@@ -73,7 +72,7 @@ def create(
cfg_scale: CFG scale for generation (default: 3.5)
steps: Number of inference steps (default: 25)
**kwargs: Additional parameters
-
+
Returns:
ImageResponse: The generated images
"""
@@ -172,7 +171,7 @@ def upload_file_alternative(img_bytes, image_format):
request_id = ''.join(random.choices(string.ascii_letters + string.digits, k=7))
message_id = ''.join(random.choices(string.ascii_letters + string.digits, k=7))
user_id = f"user_anon_{''.join(random.choices(string.digits, k=10))}"
-
+
# Generate seed if not provided
if seed is None:
seed = random.randint(0, 2**32 - 1)
@@ -215,7 +214,7 @@ def upload_file_alternative(img_bytes, image_format):
# Venice API returns binary image content directly
img_bytes = resp.content
-
+
# Convert to png or jpeg in memory
with BytesIO(img_bytes) as input_io:
with Image.open(input_io) as im:
@@ -281,7 +280,7 @@ class VeniceAI(TTICompatibleProvider):
"pony-realism-akash",
"hidream",
]
-
+
MODEL_NAMES = {
"z-image-turbo": "Z-Image Turbo",
"stable-diffusion-3.5-rev2": "Venice SD35",
@@ -299,7 +298,7 @@ def __init__(self):
self.api_endpoint = "https://outerface.venice.ai/api/inference/image"
self.session = requests.Session()
self.user_agent = LitAgent().random()
-
+
# Set up headers based on the provided request details
self.headers = {
"accept": "*/*",
@@ -320,7 +319,7 @@ def __init__(self):
"x-venice-timestamp": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
"x-venice-version": "interface@20251219.114735+e0ef642"
}
-
+
self.session.headers.update(self.headers)
self.images = Images(self)
@@ -328,23 +327,23 @@ def __init__(self):
def models(self):
"""
Get available models for the Venice AI provider.
-
+
Returns:
Object with list() method that returns available models
"""
class _ModelList:
def list(inner_self):
return type(self).AVAILABLE_MODELS
-
+
return _ModelList()
if __name__ == "__main__":
from rich import print
-
+
# Example usage
client = VeniceAI()
-
+
try:
response = client.images.create(
model="stable-diffusion-3.5-rev2",
@@ -357,4 +356,4 @@ def list(inner_self):
print("Generated image successfully:")
print(response)
except Exception as e:
- print(f"Error: {e}")
\ No newline at end of file
+ print(f"Error: {e}")
diff --git a/webscout/Provider/TTS/__init__.py b/webscout/Provider/TTS/__init__.py
index 68ca3d69..47aa4687 100644
--- a/webscout/Provider/TTS/__init__.py
+++ b/webscout/Provider/TTS/__init__.py
@@ -3,13 +3,10 @@
# Base classes
from webscout.Provider.TTS.base import (
- BaseTTSProvider,
AsyncBaseTTSProvider,
+ BaseTTSProvider,
)
-# Utility classes
-from webscout.Provider.TTS.utils import SentenceTokenizer
-
# Provider implementations
from webscout.Provider.TTS.deepgram import DeepgramTTS
from webscout.Provider.TTS.elevenlabs import ElevenlabsTTS
@@ -22,6 +19,9 @@
from webscout.Provider.TTS.speechma import SpeechMaTTS
from webscout.Provider.TTS.streamElements import StreamElements
+# Utility classes
+from webscout.Provider.TTS.utils import SentenceTokenizer
+
# List of all exported names
__all__ = [
# Base classes
diff --git a/webscout/Provider/TTS/base.py b/webscout/Provider/TTS/base.py
index 6a83c6fc..12564b89 100644
--- a/webscout/Provider/TTS/base.py
+++ b/webscout/Provider/TTS/base.py
@@ -4,31 +4,35 @@
import os
import tempfile
from pathlib import Path
-from typing import Generator, Optional, Dict, List, Union
+from typing import Generator
+
+from litprinter import ic
+
from webscout.AIbase import TTSProvider
+
class BaseTTSProvider(TTSProvider):
"""
Base class for TTS providers with OpenAI-compatible functionality.
-
+
This class implements common methods and follows OpenAI TTS API patterns
for speech generation, streaming, and audio handling.
"""
required_auth = False
-
+
# Supported models (can be overridden by subclasses)
SUPPORTED_MODELS = [
"gpt-4o-mini-tts", # Latest intelligent realtime model
"tts-1", # Lower latency model
"tts-1-hd" # Higher quality model
]
-
+
# Supported voices (can be overridden by subclasses)
SUPPORTED_VOICES = [
- "alloy", "ash", "ballad", "coral", "echo",
+ "alloy", "ash", "ballad", "coral", "echo",
"fable", "nova", "onyx", "sage", "shimmer"
]
-
+
# Supported output formats
SUPPORTED_FORMATS = [
"mp3", # Default format
@@ -38,62 +42,62 @@ class BaseTTSProvider(TTSProvider):
"wav", # Uncompressed, low latency
"pcm" # Raw samples, 24kHz 16-bit
]
-
+
def __init__(self):
"""Initialize the base TTS provider."""
self.temp_dir = tempfile.mkdtemp(prefix="webscout_tts_")
self.default_model = "gpt-4o-mini-tts"
self.default_voice = "coral"
self.default_format = "mp3"
-
+
def validate_model(self, model: str) -> str:
"""
Validate and return the model name.
-
+
Args:
model (str): Model name to validate
-
+
Returns:
str: Validated model name
-
+
Raises:
ValueError: If model is not supported
"""
# If provider doesn't support models, return the model as-is
if self.SUPPORTED_MODELS is None:
return model
-
+
if model not in self.SUPPORTED_MODELS:
raise ValueError(f"Model '{model}' not supported. Available models: {', '.join(self.SUPPORTED_MODELS)}")
return model
-
+
def validate_voice(self, voice: str) -> str:
"""
Validate and return the voice name.
-
+
Args:
voice (str): Voice name to validate
-
+
Returns:
str: Validated voice name
-
+
Raises:
ValueError: If voice is not supported
"""
if voice not in self.SUPPORTED_VOICES:
raise ValueError(f"Voice '{voice}' not supported. Available voices: {', '.join(self.SUPPORTED_VOICES)}")
return voice
-
+
def validate_format(self, response_format: str) -> str:
"""
Validate and return the response format.
-
+
Args:
response_format (str): Response format to validate
-
+
Returns:
str: Validated response format
-
+
Raises:
ValueError: If format is not supported
"""
@@ -104,54 +108,55 @@ def validate_format(self, response_format: str) -> str:
def save_audio(self, audio_file: str, destination: str = None, verbose: bool = False) -> str:
"""
Save audio to a specific destination.
-
+
Args:
audio_file (str): Path to the source audio file
destination (str, optional): Destination path. Defaults to current directory with timestamp.
verbose (bool, optional): Whether to print debug information. Defaults to False.
-
+
Returns:
str: Path to the saved audio file
-
+
Raises:
FileNotFoundError: If the audio file doesn't exist
"""
import shutil
import time
-
+
source_path = Path(audio_file)
-
+
if not source_path.exists():
raise FileNotFoundError(f"Audio file not found: {audio_file}")
-
+
if destination is None:
# Create a default destination with timestamp in current directory
timestamp = int(time.time())
destination = os.path.join(os.getcwd(), f"speech_{timestamp}{source_path.suffix}")
-
+
# Ensure the destination directory exists
os.makedirs(os.path.dirname(os.path.abspath(destination)), exist_ok=True)
-
+
# Copy the file
shutil.copy2(source_path, destination)
-
+
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Audio saved to {destination}")
-
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Audio saved to {destination}")
+
return destination
-
+
def create_speech(
- self,
- input_text: str,
+ self,
+ input_text: str,
model: str = None,
- voice: str = None,
+ voice: str = None,
response_format: str = None,
instructions: str = None,
verbose: bool = False
) -> str:
"""
Create speech from input text (OpenAI-compatible interface).
-
+
Args:
input_text (str): The text to convert to speech
model (str, optional): The TTS model to use
@@ -159,7 +164,7 @@ def create_speech(
response_format (str, optional): Audio format (mp3, opus, aac, flac, wav, pcm)
instructions (str, optional): Voice instructions for controlling speech aspects
verbose (bool, optional): Whether to print debug information
-
+
Returns:
str: Path to the generated audio file
"""
@@ -167,12 +172,12 @@ def create_speech(
model = model or self.default_model
voice = voice or self.default_voice
response_format = response_format or self.default_format
-
+
# Validate parameters
self.validate_model(model)
self.validate_voice(voice)
self.validate_format(response_format)
-
+
# Call the provider-specific TTS implementation
return self.tts(
text=input_text,
@@ -184,18 +189,18 @@ def create_speech(
)
def stream_audio(
- self,
- input_text: str,
+ self,
+ input_text: str,
model: str = None,
- voice: str = None,
+ voice: str = None,
response_format: str = None,
instructions: str = None,
- chunk_size: int = 1024,
+ chunk_size: int = 1024,
verbose: bool = False
) -> Generator[bytes, None, None]:
"""
Stream audio in chunks with OpenAI-compatible parameters.
-
+
Args:
input_text (str): The text to convert to speech
model (str, optional): The TTS model to use
@@ -204,7 +209,7 @@ def stream_audio(
instructions (str, optional): Voice instructions
chunk_size (int, optional): Size of audio chunks to yield. Defaults to 1024.
verbose (bool, optional): Whether to print debug information. Defaults to False.
-
+
Yields:
Generator[bytes, None, None]: Audio data chunks
"""
@@ -217,24 +222,24 @@ def stream_audio(
instructions=instructions,
verbose=verbose
)
-
+
# Stream the file in chunks
with open(audio_file, 'rb') as f:
while chunk := f.read(chunk_size):
yield chunk
-
+
def tts(self, text: str, **kwargs) -> str:
"""
Abstract method for text-to-speech conversion.
Must be implemented by subclasses.
-
+
Args:
text (str): The text to convert to speech
**kwargs: Additional provider-specific parameters
-
+
Returns:
str: Path to the generated audio file
-
+
Raises:
NotImplementedError: If not implemented by subclass
"""
@@ -244,25 +249,25 @@ def tts(self, text: str, **kwargs) -> str:
class AsyncBaseTTSProvider:
"""
Base class for async TTS providers with OpenAI-compatible functionality.
-
+
This class implements common async methods following OpenAI TTS API patterns
for speech generation, streaming, and audio handling.
"""
required_auth = False
-
+
# Supported models (can be overridden by subclasses)
SUPPORTED_MODELS = [
"gpt-4o-mini-tts", # Latest intelligent realtime model
"tts-1", # Lower latency model
"tts-1-hd" # Higher quality model
]
-
+
# Supported voices (can be overridden by subclasses)
SUPPORTED_VOICES = [
- "alloy", "ash", "ballad", "coral", "echo",
+ "alloy", "ash", "ballad", "coral", "echo",
"fable", "nova", "onyx", "sage", "shimmer"
]
-
+
# Supported output formats
SUPPORTED_FORMATS = [
"mp3", # Default format
@@ -272,62 +277,62 @@ class AsyncBaseTTSProvider:
"wav", # Uncompressed, low latency
"pcm" # Raw samples, 24kHz 16-bit
]
-
+
def __init__(self):
"""Initialize the async base TTS provider."""
self.temp_dir = tempfile.mkdtemp(prefix="webscout_tts_")
self.default_model = "gpt-4o-mini-tts"
self.default_voice = "coral"
self.default_format = "mp3"
-
+
async def validate_model(self, model: str) -> str:
"""
Validate and return the model name.
-
+
Args:
model (str): Model name to validate
-
+
Returns:
str: Validated model name
-
+
Raises:
ValueError: If model is not supported
"""
# If provider doesn't support models, return the model as-is
if self.SUPPORTED_MODELS is None:
return model
-
+
if model not in self.SUPPORTED_MODELS:
raise ValueError(f"Model '{model}' not supported. Available models: {', '.join(self.SUPPORTED_MODELS)}")
return model
-
+
async def validate_voice(self, voice: str) -> str:
"""
Validate and return the voice name.
-
+
Args:
voice (str): Voice name to validate
-
+
Returns:
str: Validated voice name
-
+
Raises:
ValueError: If voice is not supported
"""
if voice not in self.SUPPORTED_VOICES:
raise ValueError(f"Voice '{voice}' not supported. Available voices: {', '.join(self.SUPPORTED_VOICES)}")
return voice
-
+
async def validate_format(self, response_format: str) -> str:
"""
Validate and return the response format.
-
+
Args:
response_format (str): Response format to validate
-
+
Returns:
str: Validated response format
-
+
Raises:
ValueError: If format is not supported
"""
@@ -338,55 +343,56 @@ async def validate_format(self, response_format: str) -> str:
async def save_audio(self, audio_file: str, destination: str = None, verbose: bool = False) -> str:
"""
Save audio to a specific destination asynchronously.
-
+
Args:
audio_file (str): Path to the source audio file
destination (str, optional): Destination path. Defaults to current directory with timestamp.
verbose (bool, optional): Whether to print debug information. Defaults to False.
-
+
Returns:
str: Path to the saved audio file
-
+
Raises:
FileNotFoundError: If the audio file doesn't exist
"""
+ import asyncio
import shutil
import time
- import asyncio
-
+
source_path = Path(audio_file)
-
+
if not source_path.exists():
raise FileNotFoundError(f"Audio file not found: {audio_file}")
-
+
if destination is None:
# Create a default destination with timestamp in current directory
timestamp = int(time.time())
destination = os.path.join(os.getcwd(), f"speech_{timestamp}{source_path.suffix}")
-
+
# Ensure the destination directory exists
os.makedirs(os.path.dirname(os.path.abspath(destination)), exist_ok=True)
-
+
# Copy the file using asyncio to avoid blocking
await asyncio.to_thread(shutil.copy2, source_path, destination)
-
+
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Audio saved to {destination}")
-
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Audio saved to {destination}")
+
return destination
-
+
async def create_speech(
- self,
- input_text: str,
+ self,
+ input_text: str,
model: str = None,
- voice: str = None,
+ voice: str = None,
response_format: str = None,
instructions: str = None,
verbose: bool = False
) -> str:
"""
Create speech from input text asynchronously (OpenAI-compatible interface).
-
+
Args:
input_text (str): The text to convert to speech
model (str, optional): The TTS model to use
@@ -394,7 +400,7 @@ async def create_speech(
response_format (str, optional): Audio format (mp3, opus, aac, flac, wav, pcm)
instructions (str, optional): Voice instructions for controlling speech aspects
verbose (bool, optional): Whether to print debug information
-
+
Returns:
str: Path to the generated audio file
"""
@@ -402,12 +408,12 @@ async def create_speech(
model = model or self.default_model
voice = voice or self.default_voice
response_format = response_format or self.default_format
-
+
# Validate parameters
await self.validate_model(model)
await self.validate_voice(voice)
await self.validate_format(response_format)
-
+
# Call the provider-specific TTS implementation
return await self.tts(
text=input_text,
@@ -417,20 +423,20 @@ async def create_speech(
instructions=instructions,
verbose=verbose
)
-
+
async def stream_audio(
- self,
- input_text: str,
+ self,
+ input_text: str,
model: str = None,
- voice: str = None,
+ voice: str = None,
response_format: str = None,
instructions: str = None,
- chunk_size: int = 1024,
+ chunk_size: int = 1024,
verbose: bool = False
):
"""
Stream audio in chunks asynchronously with OpenAI-compatible parameters.
-
+
Args:
input_text (str): The text to convert to speech
model (str, optional): The TTS model to use
@@ -439,7 +445,7 @@ async def stream_audio(
instructions (str, optional): Voice instructions
chunk_size (int, optional): Size of audio chunks to yield. Defaults to 1024.
verbose (bool, optional): Whether to print debug information. Defaults to False.
-
+
Yields:
AsyncGenerator[bytes, None]: Audio data chunks
"""
@@ -447,7 +453,7 @@ async def stream_audio(
import aiofiles
except ImportError:
raise ImportError("The 'aiofiles' package is required for async streaming. Install it with 'pip install aiofiles'.")
-
+
# Generate the audio file using create_speech
audio_file = await self.create_speech(
input_text=input_text,
@@ -457,25 +463,25 @@ async def stream_audio(
instructions=instructions,
verbose=verbose
)
-
+
# Stream the file in chunks
async with aiofiles.open(audio_file, 'rb') as f:
while chunk := await f.read(chunk_size):
yield chunk
-
+
async def tts(self, text: str, **kwargs) -> str:
"""
Abstract async method for text-to-speech conversion.
Must be implemented by subclasses.
-
+
Args:
text (str): The text to convert to speech
**kwargs: Additional provider-specific parameters
-
+
Returns:
str: Path to the generated audio file
-
+
Raises:
NotImplementedError: If not implemented by subclass
"""
- raise NotImplementedError("Subclasses must implement the async tts method")
\ No newline at end of file
+ raise NotImplementedError("Subclasses must implement the async tts method")
diff --git a/webscout/Provider/TTS/deepgram.py b/webscout/Provider/TTS/deepgram.py
index 85ea52af..a0aa7d77 100644
--- a/webscout/Provider/TTS/deepgram.py
+++ b/webscout/Provider/TTS/deepgram.py
@@ -1,25 +1,24 @@
##################################################################################
## Deepgram TTS Provider ##
##################################################################################
-import time
-import requests
import pathlib
-import base64
import tempfile
-import json
-from io import BytesIO
-from webscout import exceptions
+import time
from concurrent.futures import ThreadPoolExecutor, as_completed
-from webscout.litagent import LitAgent
+
+import requests
from litprinter import ic
+from webscout import exceptions
+from webscout.litagent import LitAgent
+
try:
from . import utils
from .base import BaseTTSProvider
except ImportError:
# Handle direct execution
- import sys
import os
+ import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..'))
from webscout.Provider.TTS import utils
from webscout.Provider.TTS.base import BaseTTSProvider
@@ -27,7 +26,7 @@
class DeepgramTTS(BaseTTSProvider):
"""
Text-to-speech provider using the Deepgram Aura-2 API.
-
+
This provider follows the OpenAI TTS API structure with support for:
- Aura-2 next-gen voices
- Low-latency real-time performance
@@ -35,7 +34,7 @@ class DeepgramTTS(BaseTTSProvider):
- Concurrent generation for long texts
"""
required_auth = False
-
+
# Request headers
headers: dict[str, str] = {
"Accept": "*/*",
@@ -45,18 +44,18 @@ class DeepgramTTS(BaseTTSProvider):
"Referer": "https://deepgram.com/ai-voice-generator",
"User-Agent": LitAgent().random()
}
-
+
# Supported Aura-2 voices
SUPPORTED_MODELS = ["aura-2"]
-
+
SUPPORTED_VOICES = [
- "thalia", "odysseus", "harmonia", "theia", "electra",
+ "thalia", "odysseus", "harmonia", "theia", "electra",
"arcas", "amalthea", "helena", "hyperion", "apollo", "luna",
# Legacy Aura-1 voices (if still supported by the endpoint)
"asteria", "luna", "stella", "athena", "hera",
"zeus", "orpheus", "arcas", "perseus", "angus", "orion", "helios"
]
-
+
# Voice mapping for Deepgram API compatibility
voice_mapping = {
# Aura-2
@@ -87,7 +86,7 @@ class DeepgramTTS(BaseTTSProvider):
def __init__(self, timeout: int = 30, proxies: dict = None):
"""
Initialize the Deepgram TTS client.
-
+
Args:
timeout (int): Request timeout in seconds
proxies (dict): Proxy configuration
@@ -102,12 +101,12 @@ def __init__(self, timeout: int = 30, proxies: dict = None):
self.default_voice = "thalia"
def tts(
- self,
- text: str,
+ self,
+ text: str,
model: str = "aura-2", # Dummy model param for compatibility
- voice: str = "thalia",
+ voice: str = "thalia",
response_format: str = "mp3",
- instructions: str = None,
+ instructions: str = None,
verbose: bool = True
) -> str:
"""
@@ -124,10 +123,10 @@ def tts(
"""
if not text:
raise ValueError("Input text must be a non-empty string")
-
+
# Map voice to Deepgram API format
voice_id = self.voice_mapping.get(voice.lower(), f"aura-2-{voice.lower()}-en")
-
+
# Create temporary file
file_extension = f".{response_format}"
filename = pathlib.Path(tempfile.mktemp(suffix=file_extension, dir=self.temp_dir))
@@ -135,15 +134,17 @@ def tts(
# Split text into sentences for long inputs
sentences = utils.split_sentences(text)
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"DeepgramTTS: Processing {len(sentences)} chunks")
- ic.configureOutput(prefix='DEBUG| '); ic(f"Voice: {voice} -> {voice_id}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"DeepgramTTS: Processing {len(sentences)} chunks")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Voice: {voice} -> {voice_id}")
def generate_audio_for_chunk(part_text: str, part_number: int):
max_retries = 3
for attempt in range(max_retries):
try:
payload = {
- "text": part_text,
+ "text": part_text,
"model": voice_id,
"demoType": "voice-generator",
"params": "tag=landingpage-aivoicegenerator"
@@ -157,12 +158,14 @@ def generate_audio_for_chunk(part_text: str, part_number: int):
if response.content:
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Chunk {part_number} processed successfully")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Chunk {part_number} processed successfully")
return part_number, response.content
except requests.RequestException as e:
if verbose:
- ic.configureOutput(prefix='WARNING| '); ic(f"Error processing chunk {part_number}: {e}. Retrying {attempt+1}/{max_retries}")
+ ic.configureOutput(prefix='WARNING| ')
+ ic(f"Error processing chunk {part_number}: {e}. Retrying {attempt+1}/{max_retries}")
time.sleep(1)
raise exceptions.FailedToGenerateResponseError(f"Failed to generate audio for chunk {part_number}")
@@ -184,8 +187,9 @@ def generate_audio_for_chunk(part_text: str, part_number: int):
f.write(audio_chunks[i])
if verbose:
- ic.configureOutput(prefix='INFO| '); ic(f"Audio saved to {filename}")
-
+ ic.configureOutput(prefix='INFO| ')
+ ic(f"Audio saved to {filename}")
+
return str(filename)
except Exception as e:
diff --git a/webscout/Provider/TTS/elevenlabs.py b/webscout/Provider/TTS/elevenlabs.py
index 4c40b11e..dc30852d 100644
--- a/webscout/Provider/TTS/elevenlabs.py
+++ b/webscout/Provider/TTS/elevenlabs.py
@@ -2,23 +2,23 @@
## ElevenLabs TTS Provider ##
##################################################################################
import os
-import requests
import pathlib
import tempfile
-import time
-from io import BytesIO
-from webscout import exceptions
-from webscout.litagent import LitAgent
from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import requests
from litprinter import ic
+from webscout import exceptions
+from webscout.litagent import LitAgent
+
try:
from . import utils
from .base import BaseTTSProvider
except ImportError:
# Handle direct execution
- import sys
import os
+ import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..'))
from webscout.Provider.TTS import utils
from webscout.Provider.TTS.base import BaseTTSProvider
@@ -26,12 +26,12 @@
class ElevenlabsTTS(BaseTTSProvider):
"""
Text-to-speech provider using the ElevenLabs API.
-
- This provider supports both authenticated (with API key) and
+
+ This provider supports both authenticated (with API key) and
unauthenticated (limited) usage if available.
"""
required_auth = True
-
+
# Supported models
SUPPORTED_MODELS = [
"eleven_multilingual_v2",
@@ -41,21 +41,21 @@ class ElevenlabsTTS(BaseTTSProvider):
"eleven_turbo_v2",
"eleven_monolingual_v1"
]
-
+
# Request headers
headers: dict[str, str] = {
"User-Agent": LitAgent().random(),
"Accept": "audio/mpeg",
"Content-Type": "application/json",
}
-
+
# ElevenLabs voices
SUPPORTED_VOICES = [
- "brian", "alice", "bill", "callum", "charlie", "charlotte",
- "chris", "daniel", "eric", "george", "jessica", "laura",
+ "brian", "alice", "bill", "callum", "charlie", "charlotte",
+ "chris", "daniel", "eric", "george", "jessica", "laura",
"liam", "lily", "matilda", "sarah", "will"
]
-
+
# Voice mapping
voice_mapping = {
"brian": "nPczCjzI2devNBz1zQrb",
@@ -80,7 +80,7 @@ class ElevenlabsTTS(BaseTTSProvider):
def __init__(self, api_key: str = None, timeout: int = 30, proxies: dict = None):
"""
Initialize the ElevenLabs TTS client.
-
+
Args:
api_key (str): ElevenLabs API key. If None, tries to use unauthenticated endpoint.
timeout (int): Request timeout in seconds.
@@ -99,10 +99,10 @@ def __init__(self, api_key: str = None, timeout: int = 30, proxies: dict = None)
self.default_voice = "brian"
def tts(
- self,
- text: str,
+ self,
+ text: str,
model: str = "eleven_multilingual_v2",
- voice: str = "brian",
+ voice: str = "brian",
response_format: str = "mp3",
verbose: bool = True
) -> str:
@@ -111,15 +111,16 @@ def tts(
"""
if not text:
raise ValueError("Input text must be a non-empty string")
-
+
voice_id = self.voice_mapping.get(voice.lower(), voice)
-
+
file_extension = f".{response_format}"
filename = pathlib.Path(tempfile.mktemp(suffix=file_extension, dir=self.temp_dir))
sentences = utils.split_sentences(text)
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"ElevenlabsTTS: Processing {len(sentences)} chunks")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"ElevenlabsTTS: Processing {len(sentences)} chunks")
def generate_chunk(part_text: str, part_num: int):
payload = {
@@ -135,7 +136,7 @@ def generate_chunk(part_text: str, part_num: int):
if not self.api_key:
# Some public endpoints might still work without key but they are very restricted
params['allow_unauthenticated'] = '1'
-
+
response = self.session.post(url, json=payload, params=params, timeout=self.timeout)
if response.status_code == 401 and not self.api_key:
raise exceptions.FailedToGenerateResponseError("ElevenLabs requires an API key for this request.")
@@ -149,13 +150,14 @@ def generate_chunk(part_text: str, part_num: int):
for future in as_completed(futures):
idx, data = future.result()
audio_chunks[idx] = data
-
+
with open(filename, 'wb') as f:
for i in sorted(audio_chunks.keys()):
f.write(audio_chunks[i])
-
+
if verbose:
- ic.configureOutput(prefix='INFO| '); ic(f"Audio saved to {filename}")
+ ic.configureOutput(prefix='INFO| ')
+ ic(f"Audio saved to {filename}")
return str(filename)
except Exception as e:
raise exceptions.FailedToGenerateResponseError(f"ElevenLabs TTS failed: {e}")
diff --git a/webscout/Provider/TTS/freetts.py b/webscout/Provider/TTS/freetts.py
index cf7e9344..48aea4cd 100644
--- a/webscout/Provider/TTS/freetts.py
+++ b/webscout/Provider/TTS/freetts.py
@@ -1,28 +1,29 @@
##################################################################################
## FreeTTS Provider ##
##################################################################################
-import os
-import requests
-import time
import pathlib
import tempfile
-from datetime import datetime
-from webscout.Provider.TTS.base import BaseTTSProvider
-from webscout.litagent import LitAgent
+import time
+
+import requests
from litprinter import ic
+from webscout import exceptions
+from webscout.litagent import LitAgent
+from webscout.Provider.TTS.base import BaseTTSProvider
+
class FreeTTS(BaseTTSProvider):
"""
Text-to-speech provider using the FreeTTS.ru API.
-
+
Features:
- Multiple languages (Russian, English, Ukrainian, etc.)
- High-quality neural voices
- Supports long texts via polling
"""
required_auth = False
-
+
headers = {
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.9",
@@ -31,14 +32,14 @@ class FreeTTS(BaseTTSProvider):
"Referer": "https://freetts.ru/",
"User-Agent": LitAgent().random()
}
-
+
# Supported formats
SUPPORTED_FORMATS = ["mp3"]
def __init__(self, lang="ru-RU", timeout: int = 30, proxies: dict = None):
"""
Initialize the FreeTTS TTS client.
-
+
Args:
lang (str): Language code for voice filtering
timeout (int): Request timeout in seconds
@@ -50,13 +51,13 @@ def __init__(self, lang="ru-RU", timeout: int = 30, proxies: dict = None):
self.api_url = f"{self.base_url}/api/synthesis"
self.list_url = f"{self.base_url}/api/list"
self.history_url = f"{self.base_url}/api/history"
-
+
self.session = requests.Session()
self.session.headers.update(self.headers)
if proxies:
self.session.proxies.update(proxies)
self.timeout = timeout
-
+
self.voices = {}
self.load_voices()
# Set default voice to first available for requested lang
@@ -83,7 +84,8 @@ def load_voices(self):
if v_id not in self.SUPPORTED_VOICES:
self.SUPPORTED_VOICES.append(v_id)
except Exception as e:
- ic.configureOutput(prefix='WARNING| '); ic(f"Error loading FreeTTS voices: {e}")
+ ic.configureOutput(prefix='WARNING| ')
+ ic(f"Error loading FreeTTS voices: {e}")
def _get_default_voice(self):
for v_id, info in self.voices.items():
@@ -96,12 +98,12 @@ def get_available_voices(self):
return [v_id for v_id, info in self.voices.items() if info["lang"] == self.lang]
def tts(
- self,
- text: str,
+ self,
+ text: str,
model: str = None, # Dummy for compatibility
- voice: str = None,
+ voice: str = None,
response_format: str = "mp3",
- instructions: str = None,
+ instructions: str = None,
verbose: bool = True
) -> str:
"""
@@ -109,7 +111,7 @@ def tts(
"""
if not text:
raise ValueError("Input text must be a non-empty string")
-
+
voice = voice or self.default_voice
if not voice:
raise ValueError("No voices available")
@@ -123,28 +125,31 @@ def tts(
try:
# Step 1: Start synthesis
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"FreeTTS: Starting synthesis for voice {voice}")
-
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"FreeTTS: Starting synthesis for voice {voice}")
+
response = self.session.post(self.api_url, json=payload, timeout=self.timeout)
response.raise_for_status()
-
+
# Step 2: Poll for completion
max_polls = 20
poll_interval = 2
-
+
for i in range(max_polls):
poll_resp = self.session.get(self.api_url, timeout=self.timeout)
poll_resp.raise_for_status()
poll_data = poll_resp.json()
-
+
if poll_data.get("status") == 200 or poll_data.get("message") == "Обработка: 100%":
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"FreeTTS: Synthesis completed")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic("FreeTTS: Synthesis completed")
break
-
+
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"FreeTTS: {poll_data.get('message', 'Processing...')}")
-
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"FreeTTS: {poll_data.get('message', 'Processing...')}")
+
time.sleep(poll_interval)
else:
raise exceptions.FailedToGenerateResponseError("FreeTTS synthesis timed out")
@@ -153,7 +158,7 @@ def tts(
hist_resp = self.session.get(self.history_url, timeout=self.timeout)
hist_resp.raise_for_status()
hist_data = hist_resp.json()
-
+
if hist_data.get("status") == "success":
history = hist_data.get("data", [])
if history:
@@ -163,18 +168,19 @@ def tts(
if audio_url:
if not audio_url.startswith("http"):
audio_url = self.base_url + audio_url
-
+
# Download the file
audio_file_resp = self.session.get(audio_url, timeout=self.timeout)
audio_file_resp.raise_for_status()
-
+
# Save to temp file
filename = pathlib.Path(tempfile.mktemp(suffix=f".{response_format}", dir=self.temp_dir))
with open(filename, "wb") as f:
f.write(audio_file_resp.content)
-
+
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"FreeTTS: Audio saved to {filename}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"FreeTTS: Audio saved to {filename}")
return str(filename)
raise exceptions.FailedToGenerateResponseError("Failed to retrieve audio URL from history")
diff --git a/webscout/Provider/TTS/murfai.py b/webscout/Provider/TTS/murfai.py
index 45dd341a..3f88d6a0 100644
--- a/webscout/Provider/TTS/murfai.py
+++ b/webscout/Provider/TTS/murfai.py
@@ -1,12 +1,16 @@
-import time
-import requests
import pathlib
import tempfile
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
from io import BytesIO
from urllib.parse import urlencode
+
+import requests
+from litprinter import ic
+
from webscout import exceptions
from webscout.litagent import LitAgent
-from concurrent.futures import ThreadPoolExecutor, as_completed
+
from . import utils
from .base import BaseTTSProvider
@@ -14,7 +18,7 @@
class MurfAITTS(BaseTTSProvider):
"""
Text-to-speech provider using the MurfAITTS API with OpenAI-compatible interface.
-
+
This provider follows the OpenAI TTS API structure with support for:
- Multiple TTS models (gpt-4o-mini-tts, tts-1, tts-1-hd)
- Multiple voices with OpenAI-style naming
@@ -23,26 +27,26 @@ class MurfAITTS(BaseTTSProvider):
- Streaming support
"""
required_auth = False
-
+
# Override supported models for MurfAI (set to None as requested)
SUPPORTED_MODELS = None
-
+
# Override supported voices with real MurfAI voice names
SUPPORTED_VOICES = [
"Hazel", "Marcus", "Samantha", "Natalie", "Michelle", "Ken", "Clint", "Amit", "Priya"
]
-
+
# Override supported formats
SUPPORTED_FORMATS = [
"mp3", # Default format for MurfAI
"wav" # Alternative format
]
-
+
# Request headers
headers: dict[str, str] = {
"User-Agent": LitAgent().random()
}
-
+
# Voice mapping from real names to MurfAI voice IDs
voice_mapping: dict[str, str] = {
"Hazel": "en-UK-hazel",
@@ -68,13 +72,13 @@ def __init__(self, timeout: int = 20, proxies: dict = None):
def tts(self, text: str, voice: str = "Hazel", verbose: bool = False, **kwargs) -> str:
"""
Converts text to speech using the MurfAITTS API and saves it to a file.
-
+
Args:
text (str): The text to convert to speech
voice (str): The voice to use (default: "Hazel")
verbose (bool): Whether to print debug information
**kwargs: Additional parameters (model, response_format, instructions)
-
+
Returns:
str: Path to the generated audio file
"""
@@ -83,21 +87,21 @@ def tts(self, text: str, voice: str = "Hazel", verbose: bool = False, **kwargs)
raise ValueError("Input text must be a non-empty string")
if len(text) > 10000:
raise ValueError("Input text exceeds maximum allowed length of 10,000 characters")
-
+
# Use default voice if not provided
if voice is None:
voice = "Hazel"
-
+
# Validate voice using base class method
self.validate_voice(voice)
-
+
# Map real voice name to MurfAI voice ID
voice_id = self.voice_mapping.get(voice, "en-UK-hazel") # Default to Hazel
-
+
# Get response format from kwargs or use default
response_format = kwargs.get('response_format', 'mp3')
response_format = self.validate_format(response_format)
-
+
# Create temporary file with appropriate extension
file_extension = f".{response_format}" if response_format != "pcm" else ".wav"
filename = pathlib.Path(tempfile.mktemp(suffix=file_extension, dir=self.temp_dir))
@@ -120,14 +124,17 @@ def generate_audio_for_chunk(part_text: str, part_number: int):
# Check if the request was successful
if response.ok and response.status_code == 200:
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Chunk {part_number} processed successfully")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Chunk {part_number} processed successfully")
return part_number, response.content
else:
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"No data received for chunk {part_number}. Retrying...")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"No data received for chunk {part_number}. Retrying...")
except requests.RequestException as e:
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Error for chunk {part_number}: {e}. Retrying...")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Error for chunk {part_number}: {e}. Retrying...")
time.sleep(1)
try:
# Using ThreadPoolExecutor to handle requests concurrently
@@ -145,25 +152,29 @@ def generate_audio_for_chunk(part_text: str, part_number: int):
audio_chunks[part_number] = audio_data # Store the audio data in correct sequence
except Exception as e:
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Failed to generate audio for chunk {chunk_num}: {e}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Failed to generate audio for chunk {chunk_num}: {e}")
# Combine audio chunks in the correct sequence
combined_audio = BytesIO()
for part_number in sorted(audio_chunks.keys()):
combined_audio.write(audio_chunks[part_number])
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Added chunk {part_number} to the combined file.")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Added chunk {part_number} to the combined file.")
# Save the combined audio data to a single file
with open(filename, 'wb') as f:
f.write(combined_audio.getvalue())
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Final Audio Saved as {filename}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Final Audio Saved as {filename}")
return filename.as_posix()
except requests.exceptions.RequestException as e:
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Failed to perform the operation: {e}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Failed to perform the operation: {e}")
raise exceptions.FailedToGenerateResponseError(
f"Failed to perform the operation: {e}"
)
@@ -171,7 +182,7 @@ def generate_audio_for_chunk(part_text: str, part_number: int):
def create_speech(
self,
input: str,
- model: str = "gpt-4o-mini-tts",
+ model: str = "gpt-4o-mini-tts",
voice: str = "Hazel",
response_format: str = "mp3",
instructions: str = None,
@@ -179,7 +190,7 @@ def create_speech(
) -> str:
"""
OpenAI-compatible speech creation interface.
-
+
Args:
input (str): The text to convert to speech
model (str): The TTS model to use
@@ -187,7 +198,7 @@ def create_speech(
response_format (str): Audio format
instructions (str): Voice instructions (not used by MurfAI)
verbose (bool): Whether to print debug information
-
+
Returns:
str: Path to the generated audio file
"""
@@ -210,7 +221,7 @@ def stream_audio(
):
"""
Stream audio response in chunks.
-
+
Args:
input (str): The text to convert to speech
model (str): The TTS model to use
@@ -219,7 +230,7 @@ def stream_audio(
instructions (str): Voice instructions
chunk_size (int): Size of audio chunks to yield
verbose (bool): Whether to print debug information
-
+
Yields:
bytes: Audio data chunks
"""
@@ -232,7 +243,7 @@ def stream_audio(
instructions=instructions,
verbose=verbose
)
-
+
# Stream the file in chunks
with open(audio_file, 'rb') as f:
while chunk := f.read(chunk_size):
@@ -243,7 +254,8 @@ def stream_audio(
murfai = MurfAITTS()
text = "This is a test of the MurfAITTS text-to-speech API. It supports multiple sentences and advanced logging."
- ic.configureOutput(prefix='DEBUG| '); ic("Generating audio...")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic("Generating audio...")
try:
audio_file = murfai.create_speech(
input=text,
@@ -252,6 +264,8 @@ def stream_audio(
response_format="mp3",
verbose=True
)
- ic.configureOutput(prefix='INFO| '); ic(f"Audio saved to: {audio_file}")
+ ic.configureOutput(prefix='INFO| ')
+ ic(f"Audio saved to: {audio_file}")
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Error: {e}")
\ No newline at end of file
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Error: {e}")
diff --git a/webscout/Provider/TTS/openai_fm.py b/webscout/Provider/TTS/openai_fm.py
index fbe80142..a5fc4857 100644
--- a/webscout/Provider/TTS/openai_fm.py
+++ b/webscout/Provider/TTS/openai_fm.py
@@ -1,21 +1,21 @@
##################################################################################
## OpenAI.fm TTS Provider ##
##################################################################################
-import time
-import requests
import pathlib
import tempfile
-from io import BytesIO
+
+import requests
+from litprinter import ic
+
from webscout import exceptions
from webscout.litagent import LitAgent
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from webscout.Provider.TTS import utils
from webscout.Provider.TTS.base import BaseTTSProvider
+
class OpenAIFMTTS(BaseTTSProvider):
"""
Text-to-speech provider using the OpenAI.fm API with OpenAI-compatible interface.
-
+
This provider follows the OpenAI TTS API structure with support for:
- Multiple TTS models (gpt-4o-mini-tts, tts-1, tts-1-hd)
- 11 built-in voices optimized for English
@@ -24,7 +24,7 @@ class OpenAIFMTTS(BaseTTSProvider):
- Streaming support
"""
required_auth = False
-
+
# Request headers
headers = {
"accept": "*/*",
@@ -37,14 +37,14 @@ class OpenAIFMTTS(BaseTTSProvider):
"user-agent": LitAgent().random(),
"referer": "https://www.openai.fm"
}
-
+
# Override supported models for OpenAI.fm
SUPPORTED_MODELS = [
"gpt-4o-mini-tts", # Latest intelligent realtime model
- "tts-1", # Lower latency model
+ "tts-1", # Lower latency model
"tts-1-hd" # Higher quality model
]
-
+
# OpenAI.fm supported voices (11 built-in voices)
SUPPORTED_VOICES = [
"alloy", # Neutral voice with balanced tone
@@ -58,11 +58,11 @@ class OpenAIFMTTS(BaseTTSProvider):
"sage", # Measured and contemplative voice
"shimmer" # Bright and optimistic voice
]
-
+
# Voice mapping for API compatibility
voice_mapping = {
"alloy": "alloy",
- "ash": "ash",
+ "ash": "ash",
"ballad": "ballad",
"coral": "coral",
"echo": "echo",
@@ -76,7 +76,7 @@ class OpenAIFMTTS(BaseTTSProvider):
def __init__(self, timeout: int = 20, proxies: dict = None):
"""
Initialize the OpenAI.fm TTS client.
-
+
Args:
timeout (int): Request timeout in seconds
proxies (dict): Proxy configuration
@@ -90,12 +90,12 @@ def __init__(self, timeout: int = 20, proxies: dict = None):
self.timeout = timeout
def tts(
- self,
- text: str,
+ self,
+ text: str,
model: str = "gpt-4o-mini-tts",
- voice: str = "coral",
+ voice: str = "coral",
response_format: str = "mp3",
- instructions: str = None,
+ instructions: str = None,
verbose: bool = True
) -> str:
"""
@@ -121,24 +121,24 @@ def tts(
raise ValueError("Input text must be a non-empty string")
if len(text) > 10000:
raise ValueError("Input text exceeds maximum allowed length of 10,000 characters")
-
+
# Validate model, voice, and format using base class methods
model = self.validate_model(model)
voice = self.validate_voice(voice)
response_format = self.validate_format(response_format)
-
+
# Map voice to API format
voice_id = self.voice_mapping.get(voice, voice)
-
+
# Set default instructions if not provided
if instructions is None:
instructions = "Speak in a cheerful and positive tone."
-
+
# Create temporary file with appropriate extension
file_extension = f".{response_format}" if response_format != "pcm" else ".wav"
with tempfile.NamedTemporaryFile(suffix=file_extension, dir=self.temp_dir, delete=False) as temp_file:
filename = pathlib.Path(temp_file.name)
-
+
# Prepare parameters for the API request
params = {
"input": text,
@@ -156,33 +156,40 @@ def tts(
timeout=self.timeout
)
response.raise_for_status()
-
+
# Validate response content
if not response.content:
raise exceptions.FailedToGenerateResponseError("Empty response from API")
-
+
# Save the audio file
with open(filename, "wb") as f:
f.write(response.content)
-
+
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic("Speech generated successfully")
- ic.configureOutput(prefix='DEBUG| '); ic(f"Model: {model}")
- ic.configureOutput(prefix='DEBUG| '); ic(f"Voice: {voice}")
- ic.configureOutput(prefix='DEBUG| '); ic(f"Format: {response_format}")
- ic.configureOutput(prefix='DEBUG| '); ic(f"Audio saved to {filename}")
-
+ ic.configureOutput(prefix='DEBUG| ')
+ ic("Speech generated successfully")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Model: {model}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Voice: {voice}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Format: {response_format}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Audio saved to {filename}")
+
return filename.as_posix()
-
+
except requests.exceptions.RequestException as e:
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Failed to generate speech: {e}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Failed to generate speech: {e}")
raise exceptions.FailedToGenerateResponseError(
f"Failed to generate speech: {e}"
)
except Exception as e:
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Unexpected error: {e}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Unexpected error: {e}")
raise exceptions.FailedToGenerateResponseError(
f"Unexpected error during speech generation: {e}"
)
@@ -190,7 +197,7 @@ def tts(
def create_speech(
self,
input: str,
- model: str = "gpt-4o-mini-tts",
+ model: str = "gpt-4o-mini-tts",
voice: str = "coral",
response_format: str = "mp3",
instructions: str = None,
@@ -198,7 +205,7 @@ def create_speech(
) -> str:
"""
OpenAI-compatible speech creation interface.
-
+
Args:
input (str): The text to convert to speech
model (str): The TTS model to use
@@ -206,7 +213,7 @@ def create_speech(
response_format (str): Audio format
instructions (str): Voice instructions
verbose (bool): Whether to print debug information
-
+
Returns:
str: Path to the generated audio file
"""
@@ -222,7 +229,7 @@ def create_speech(
def with_streaming_response(self):
"""
Return a streaming response context manager (OpenAI-compatible).
-
+
Returns:
StreamingResponseContextManager: Context manager for streaming responses
"""
@@ -233,29 +240,29 @@ class StreamingResponseContextManager:
"""
Context manager for streaming TTS responses (OpenAI-compatible).
"""
-
+
def __init__(self, tts_provider: OpenAIFMTTS):
self.tts_provider = tts_provider
self.audio_file = None
-
+
def create(
self,
input: str,
model: str = "gpt-4o-mini-tts",
- voice: str = "coral",
+ voice: str = "coral",
response_format: str = "mp3",
instructions: str = None
):
"""
Create speech with streaming capability.
-
+
Args:
input (str): The text to convert to speech
model (str): The TTS model to use
voice (str): The voice to use
response_format (str): Audio format
instructions (str): Voice instructions
-
+
Returns:
StreamingResponse: Streaming response object
"""
@@ -267,10 +274,10 @@ def create(
instructions=instructions
)
return StreamingResponse(self.audio_file)
-
+
def __enter__(self):
return self
-
+
def __exit__(self, exc_type, exc_val, exc_tb):
pass
@@ -279,36 +286,36 @@ class StreamingResponse:
"""
Streaming response object for TTS audio (OpenAI-compatible).
"""
-
+
def __init__(self, audio_file: str):
self.audio_file = audio_file
-
+
def __enter__(self):
"""Enter the context manager."""
return self
-
+
def __exit__(self, exc_type, exc_val, exc_tb):
"""Exit the context manager."""
pass
-
+
def stream_to_file(self, file_path: str, chunk_size: int = 1024):
"""
Stream audio content to a file.
-
+
Args:
file_path (str): Destination file path
chunk_size (int): Size of chunks to read/write
"""
import shutil
shutil.copy2(self.audio_file, file_path)
-
+
def iter_bytes(self, chunk_size: int = 1024):
"""
Iterate over audio bytes in chunks.
-
+
Args:
chunk_size (int): Size of chunks to yield
-
+
Yields:
bytes: Audio data chunks
"""
@@ -320,10 +327,11 @@ def iter_bytes(self, chunk_size: int = 1024):
if __name__ == "__main__":
# Example usage demonstrating OpenAI-compatible interface
tts_provider = OpenAIFMTTS()
-
+
try:
# Basic usage
- ic.configureOutput(prefix='DEBUG| '); ic("Testing basic speech generation...")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic("Testing basic speech generation...")
audio_file = tts_provider.create_speech(
input="Today is a wonderful day to build something people love!",
model="gpt-4o-mini-tts",
@@ -331,18 +339,22 @@ def iter_bytes(self, chunk_size: int = 1024):
instructions="Speak in a cheerful and positive tone."
)
print(f"Audio file generated: {audio_file}")
-
+
# Streaming usage
- ic.configureOutput(prefix='DEBUG| '); ic("Testing streaming response...")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic("Testing streaming response...")
with tts_provider.with_streaming_response().create(
input="This is a streaming test.",
voice="alloy",
response_format="wav"
) as response:
response.stream_to_file("streaming_test.wav")
- ic.configureOutput(prefix='INFO| '); ic("Streaming audio saved to streaming_test.wav")
-
+ ic.configureOutput(prefix='INFO| ')
+ ic("Streaming audio saved to streaming_test.wav")
+
except exceptions.FailedToGenerateResponseError as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Error: {e}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Error: {e}")
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Unexpected error: {e}")
\ No newline at end of file
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Unexpected error: {e}")
diff --git a/webscout/Provider/TTS/parler.py b/webscout/Provider/TTS/parler.py
index 00f8835c..89f88d28 100644
--- a/webscout/Provider/TTS/parler.py
+++ b/webscout/Provider/TTS/parler.py
@@ -2,13 +2,15 @@
## ParlerTTS Provider ##
##################################################################################
import json
+import pathlib
import random
import string
-import time
-import pathlib
import tempfile
+from typing import Optional
+
import httpx
-from typing import Optional, Union, Dict, List
+from litprinter import ic
+
from webscout import exceptions
from webscout.litagent import LitAgent
@@ -17,32 +19,31 @@
from .base import BaseTTSProvider
except ImportError:
# Handle direct execution
- import sys
import os
+ import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..'))
- from webscout.Provider.TTS import utils
from webscout.Provider.TTS.base import BaseTTSProvider
class ParlerTTS(BaseTTSProvider):
"""
Text-to-speech provider using the Parler-TTS API (Hugging Face Spaces).
-
+
Features:
- High-fidelity speech generation
- Controllable via simple text prompts (description)
- Manual polling logic for robustness
"""
required_auth = False
-
+
BASE_URL = "https://parler-tts-parler-tts.hf.space"
-
+
# Request headers
headers: dict[str, str] = {
"User-Agent": LitAgent().random(),
"origin": BASE_URL,
"referer": f"{BASE_URL}/",
}
-
+
SUPPORTED_MODELS = ["parler-mini-v1", "parler-large-v1"]
def __init__(self, timeout: int = 120, proxy: Optional[str] = None):
@@ -58,8 +59,8 @@ def _generate_session_hash(self) -> str:
return "".join(random.choices(string.ascii_lowercase + string.digits, k=10))
def tts(
- self,
- text: str,
+ self,
+ text: str,
description: str = "A female speaker delivers a slightly expressive and animated speech with a moderate speed. The recording features a low-pitch voice and very clear audio.",
use_large: bool = False,
response_format: str = "wav",
@@ -70,16 +71,18 @@ def tts(
"""
if not text:
raise ValueError("Input text must be a non-empty string")
-
+
session_hash = self._generate_session_hash()
filename = pathlib.Path(tempfile.mktemp(suffix=f".{response_format}", dir=self.temp_dir))
-
+
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"ParlerTTS: Generating speech for '{text[:20]}...'")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"ParlerTTS: Generating speech for '{text[:20]}...'")
client_kwargs = {"headers": self.headers, "timeout": self.timeout}
- if self.proxy: client_kwargs["proxy"] = self.proxy
-
+ if self.proxy:
+ client_kwargs["proxy"] = self.proxy
+
try:
with httpx.Client(**client_kwargs) as client:
# Step 1: Join the queue
@@ -92,23 +95,25 @@ def tts(
"trigger_id": 8,
"session_hash": session_hash
}
-
+
response = client.post(join_url, json=payload)
response.raise_for_status()
-
+
# Step 2: Poll for data
data_url = f"{self.BASE_URL}/queue/data?session_hash={session_hash}"
audio_url = None
-
+
# Gradio Spaces can take time to wake up or process
with client.stream("GET", data_url) as stream:
for line in stream.iter_lines():
- if not line: continue
+ if not line:
+ continue
if line.startswith("data: "):
try:
data = json.loads(line[6:])
- except json.JSONDecodeError: continue
-
+ except json.JSONDecodeError:
+ continue
+
msg = data.get("msg")
if msg == "process_completed":
if data.get("success"):
@@ -132,17 +137,20 @@ def tts(
# Step 3: Download the audio file
audio_response = client.get(audio_url)
audio_response.raise_for_status()
-
+
with open(filename, "wb") as f:
f.write(audio_response.content)
-
+
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Speech generated successfully: {filename}")
-
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Speech generated successfully: {filename}")
+
return filename.as_posix()
except Exception as e:
- if verbose: ic.configureOutput(prefix='DEBUG| '); ic(f"Error in ParlerTTS: {e}")
+ if verbose:
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Error in ParlerTTS: {e}")
raise exceptions.FailedToGenerateResponseError(f"Failed to generate audio: {e}")
def create_speech(self, input: str, **kwargs) -> str:
@@ -152,6 +160,8 @@ def create_speech(self, input: str, **kwargs) -> str:
tts = ParlerTTS()
try:
path = tts.tts("Testing Parler-TTS with manual polling.", verbose=True)
- ic.configureOutput(prefix='INFO| '); ic(f"Saved to {path}")
+ ic.configureOutput(prefix='INFO| ')
+ ic(f"Saved to {path}")
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Error: {e}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Error: {e}")
diff --git a/webscout/Provider/TTS/qwen.py b/webscout/Provider/TTS/qwen.py
index fe1be83f..1848bd9a 100644
--- a/webscout/Provider/TTS/qwen.py
+++ b/webscout/Provider/TTS/qwen.py
@@ -2,13 +2,15 @@
## Qwen3-TTS Provider ##
##################################################################################
import json
+import pathlib
import random
import string
-import time
-import pathlib
import tempfile
+from typing import Optional
+
import httpx
-from typing import Optional, Union, Dict, List
+from litprinter import ic
+
from webscout import exceptions
from webscout.litagent import LitAgent
@@ -17,16 +19,15 @@
from .base import BaseTTSProvider
except ImportError:
# Handle direct execution
- import sys
import os
+ import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..'))
- from webscout.Provider.TTS import utils
from webscout.Provider.TTS.base import BaseTTSProvider
class QwenTTS(BaseTTSProvider):
"""
Text-to-speech provider using the Qwen3-TTS API (Hugging Face Spaces).
-
+
This provider follows the OpenAI TTS API structure with support for:
- Multiple TTS models (mapped to Gradio fn_index)
- 40+ high-quality voices across multiple languages
@@ -35,19 +36,19 @@ class QwenTTS(BaseTTSProvider):
- Streaming response simulation
"""
required_auth = False
-
+
BASE_URL = "https://qwen-qwen3-tts-demo.hf.space"
-
+
# Request headers
headers: dict[str, str] = {
"User-Agent": LitAgent().random(),
"origin": BASE_URL,
"referer": f"{BASE_URL}/",
}
-
+
# Override supported models
SUPPORTED_MODELS = ["qwen3-tts"]
-
+
# Supported voices
SUPPORTED_VOICES = [
"cherry", "serena", "ethan", "chelsie", "momo", "vivian", "moon", "maia",
@@ -57,7 +58,7 @@ class QwenTTS(BaseTTSProvider):
"neil", "elias", "arthur", "nini", "ebona", "seren", "pip", "stella", "li",
"marcus", "roy", "peter", "eric", "rocky", "kiki", "sunny", "jada", "dylan"
]
-
+
# Voice mapping for API compatibility
voice_mapping = {
"cherry": "Cherry / 芊悦",
@@ -114,7 +115,7 @@ class QwenTTS(BaseTTSProvider):
def __init__(self, timeout: int = 60, proxy: Optional[str] = None):
"""
Initialize the QwenTTS client.
-
+
Args:
timeout (int): Request timeout in seconds
proxy (str): Proxy configuration string
@@ -130,10 +131,10 @@ def _generate_session_hash(self) -> str:
return "".join(random.choices(string.ascii_lowercase + string.digits, k=10))
def tts(
- self,
- text: str,
+ self,
+ text: str,
model: str = "qwen3-tts",
- voice: str = "cherry",
+ voice: str = "cherry",
response_format: str = "wav",
language: str = "Auto / 自动",
verbose: bool = True
@@ -158,26 +159,27 @@ def tts(
"""
if not text or not isinstance(text, str):
raise ValueError("Input text must be a non-empty string")
-
+
voice = self.validate_voice(voice)
qwen_voice = self.voice_mapping.get(voice, self.voice_mapping["cherry"])
-
+
# Create temporary file
file_extension = f".{response_format}"
filename = pathlib.Path(tempfile.mktemp(suffix=file_extension, dir=self.temp_dir))
-
+
session_hash = self._generate_session_hash()
-
+
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Joining queue for voice: {voice} ({qwen_voice})")
-
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Joining queue for voice: {voice} ({qwen_voice})")
+
client_kwargs = {
"headers": self.headers,
"timeout": self.timeout
}
if self.proxy:
client_kwargs["proxy"] = self.proxy
-
+
try:
with httpx.Client(**client_kwargs) as client:
# Step 1: Join the queue
@@ -189,23 +191,24 @@ def tts(
"trigger_id": 7,
"session_hash": session_hash
}
-
+
response = client.post(join_url, json=payload)
response.raise_for_status()
-
+
# Step 2: Poll for data (SSE)
data_url = f"{self.BASE_URL}/gradio_api/queue/data?session_hash={session_hash}"
audio_url = None
-
+
with client.stream("GET", data_url) as stream:
for line in stream.iter_lines():
- if not line: continue
+ if not line:
+ continue
if line.startswith("data: "):
try:
data = json.loads(line[6:])
except json.JSONDecodeError:
continue
-
+
msg = data.get("msg")
if msg == "process_completed":
if data.get("success"):
@@ -226,24 +229,26 @@ def tts(
# Step 3: Download the audio file
audio_response = client.get(audio_url)
audio_response.raise_for_status()
-
+
with open(filename, "wb") as f:
f.write(audio_response.content)
-
+
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Speech generated successfully: {filename}")
-
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Speech generated successfully: {filename}")
+
return filename.as_posix()
except Exception as e:
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Error in QwenTTS: {e}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Error in QwenTTS: {e}")
raise exceptions.FailedToGenerateResponseError(f"Failed to generate audio: {e}")
def create_speech(
self,
input: str,
- model: str = "qwen3-tts",
+ model: str = "qwen3-tts",
voice: str = "cherry",
response_format: str = "wav",
verbose: bool = False,
@@ -267,11 +272,11 @@ class StreamingResponseContextManager:
"""Context manager for streaming TTS responses."""
def __init__(self, tts_provider: QwenTTS):
self.tts_provider = tts_provider
-
+
def create(self, **kwargs):
audio_file = self.tts_provider.create_speech(**kwargs)
return StreamingResponse(audio_file)
-
+
def __enter__(self): return self
def __exit__(self, exc_type, exc_val, exc_tb): pass
@@ -279,14 +284,14 @@ class StreamingResponse:
"""Streaming response object for TTS audio."""
def __init__(self, audio_file: str):
self.audio_file = audio_file
-
+
def __enter__(self): return self
def __exit__(self, exc_type, exc_val, exc_tb): pass
-
+
def stream_to_file(self, file_path: str):
import shutil
shutil.copy2(self.audio_file, file_path)
-
+
def iter_bytes(self, chunk_size: int = 1024):
with open(self.audio_file, 'rb') as f:
while chunk := f.read(chunk_size):
@@ -295,8 +300,11 @@ def iter_bytes(self, chunk_size: int = 1024):
if __name__ == "__main__":
qwen = QwenTTS()
try:
- ic.configureOutput(prefix='DEBUG| '); ic("Testing Qwen3-TTS...")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic("Testing Qwen3-TTS...")
path = qwen.create_speech(input="Hello, this is a test.", voice="jennifer", verbose=True)
- ic.configureOutput(prefix='INFO| '); ic(f"Saved to {path}")
+ ic.configureOutput(prefix='INFO| ')
+ ic(f"Saved to {path}")
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Error: {e}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Error: {e}")
diff --git a/webscout/Provider/TTS/sherpa.py b/webscout/Provider/TTS/sherpa.py
index fd36770f..4bb7be4b 100644
--- a/webscout/Provider/TTS/sherpa.py
+++ b/webscout/Provider/TTS/sherpa.py
@@ -2,13 +2,15 @@
## SherpaTTS Provider ##
##################################################################################
import json
+import pathlib
import random
import string
-import time
-import pathlib
import tempfile
+from typing import Optional
+
import httpx
-from typing import Optional, Union, Dict, List
+from litprinter import ic
+
from webscout import exceptions
from webscout.litagent import LitAgent
@@ -17,16 +19,15 @@
from .base import BaseTTSProvider
except ImportError:
# Handle direct execution
- import sys
import os
+ import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..'))
- from webscout.Provider.TTS import utils
from webscout.Provider.TTS.base import BaseTTSProvider
class SherpaTTS(BaseTTSProvider):
"""
Text-to-speech provider using the Next-gen Kaldi (Sherpa-ONNX) API.
-
+
This provider follows the OpenAI TTS API structure with support for:
- 50+ languages including English, Chinese, Cantonese, Arabic, French, etc.
- Multiple ONNX-based models (Kokoro, Piper, Coqui, etc.)
@@ -34,16 +35,16 @@ class SherpaTTS(BaseTTSProvider):
- Multiple output formats
"""
required_auth = False
-
+
BASE_URL = "https://k2-fsa-text-to-speech.hf.space"
-
+
# Request headers
headers: dict[str, str] = {
"User-Agent": LitAgent().random(),
"origin": BASE_URL,
"referer": f"{BASE_URL}/",
}
-
+
SUPPORTED_MODELS = [
"csukuangfj/kokoro-en-v0_19|11 speakers",
"csukuangfj/kitten-kitten-en-v0_1-fp16|8 speakers",
@@ -96,7 +97,7 @@ class SherpaTTS(BaseTTSProvider):
"csukuangfj/vits-vctk|109 speakers",
"csukuangfj/vits-ljs|1 speaker"
]
-
+
LANGUAGES = [
"English", "Chinese (Mandarin, 普通话)", "Chinese+English", "Persian+English",
"Cantonese (粤语)", "Min-nan (闽南话)", "Arabic", "Afrikaans", "Bengali",
@@ -123,8 +124,8 @@ def _generate_session_hash(self) -> str:
return "".join(random.choices(string.ascii_lowercase + string.digits, k=11))
def tts(
- self,
- text: str,
+ self,
+ text: str,
language: str = "English",
model_choice: str = "csukuangfj/kokoro-en-v0_19|11 speakers",
speaker_id: str = "0",
@@ -146,18 +147,20 @@ def tts(
"""
if not text:
raise ValueError("Input text must be a non-empty string")
-
+
model_choice = self.validate_model(model_choice)
-
+
session_hash = self._generate_session_hash()
filename = pathlib.Path(tempfile.mktemp(suffix=f".{response_format}", dir=self.temp_dir))
-
+
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"SherpaTTS: Generating speech for '{text[:20]}...' using {language}/{model_choice}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"SherpaTTS: Generating speech for '{text[:20]}...' using {language}/{model_choice}")
client_kwargs = {"headers": self.headers, "timeout": self.timeout}
- if self.proxy: client_kwargs["proxy"] = self.proxy
-
+ if self.proxy:
+ client_kwargs["proxy"] = self.proxy
+
try:
with httpx.Client(**client_kwargs) as client:
# Step 1: Join the queue
@@ -169,22 +172,24 @@ def tts(
"trigger_id": 9,
"session_hash": session_hash
}
-
+
response = client.post(join_url, json=payload)
response.raise_for_status()
-
+
# Step 2: Poll for data
data_url = f"{self.BASE_URL}/gradio_api/queue/data?session_hash={session_hash}"
audio_url = None
-
+
with client.stream("GET", data_url) as stream:
for line in stream.iter_lines():
- if not line: continue
+ if not line:
+ continue
if line.startswith("data: "):
try:
data = json.loads(line[6:])
- except json.JSONDecodeError: continue
-
+ except json.JSONDecodeError:
+ continue
+
msg = data.get("msg")
if msg == "process_completed":
if data.get("success"):
@@ -205,17 +210,20 @@ def tts(
# Step 3: Download the audio file
audio_response = client.get(audio_url)
audio_response.raise_for_status()
-
+
with open(filename, "wb") as f:
f.write(audio_response.content)
-
+
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Speech generated successfully: {filename}")
-
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Speech generated successfully: {filename}")
+
return filename.as_posix()
except Exception as e:
- if verbose: ic.configureOutput(prefix='DEBUG| '); ic(f"Error in SherpaTTS: {e}")
+ if verbose:
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Error in SherpaTTS: {e}")
raise exceptions.FailedToGenerateResponseError(f"Failed to generate audio: {e}")
def create_speech(self, input: str, **kwargs) -> str:
@@ -251,6 +259,8 @@ def iter_bytes(self, chunk_size: int = 1024):
tts = SherpaTTS()
try:
path = tts.tts("This is a Sherpa-ONNX test.", verbose=True)
- ic.configureOutput(prefix='INFO| '); ic(f"Result: {path}")
+ ic.configureOutput(prefix='INFO| ')
+ ic(f"Result: {path}")
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Error: {e}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Error: {e}")
diff --git a/webscout/Provider/TTS/speechma.py b/webscout/Provider/TTS/speechma.py
index bed1120d..5623dc13 100644
--- a/webscout/Provider/TTS/speechma.py
+++ b/webscout/Provider/TTS/speechma.py
@@ -1,23 +1,25 @@
##################################################################################
## Modified version of code written by t.me/infip1217 ##
##################################################################################
-import time
-import requests
import pathlib
import tempfile
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
from io import BytesIO
+
+import requests
+from litprinter import ic
+
from webscout import exceptions
from webscout.litagent import LitAgent
-from litprinter import ic
-from concurrent.futures import ThreadPoolExecutor, as_completed
try:
from . import utils
from .base import BaseTTSProvider
except ImportError:
# Handle direct execution
- import sys
import os
+ import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..'))
from webscout.Provider.TTS import utils
from webscout.Provider.TTS.base import BaseTTSProvider
@@ -25,7 +27,7 @@
class SpeechMaTTS(BaseTTSProvider):
"""
Text-to-speech provider using the SpeechMa API with OpenAI-compatible interface.
-
+
This provider follows the OpenAI TTS API structure with support for:
- Multiple TTS models (gpt-4o-mini-tts, tts-1, tts-1-hd)
- Multilingual voices with pitch and rate control
@@ -34,7 +36,7 @@ class SpeechMaTTS(BaseTTSProvider):
- Streaming support
"""
required_auth = False
-
+
# Request headers
headers = {
"authority": "speechma.com",
@@ -43,10 +45,10 @@ class SpeechMaTTS(BaseTTSProvider):
"content-type": "application/json",
**LitAgent().generate_fingerprint()
}
-
+
# SpeechMa doesn't support different models - set to None
SUPPORTED_MODELS = None
-
+
# All supported voices from SpeechMa API
SUPPORTED_VOICES = [
"aditi", "amy", "astrid", "bianca", "carla", "carmen", "celine", "chant",
@@ -77,7 +79,7 @@ class SpeechMaTTS(BaseTTSProvider):
"michelle", "roger", "libby", "ryan", "sonia", "thomas", "natasha",
"william", "clara", "liam"
]
-
+
# Voice mapping for SpeechMa API compatibility (lowercase keys for all voices)
voice_mapping = {
# Standard voices
@@ -236,7 +238,7 @@ class SpeechMaTTS(BaseTTSProvider):
def __init__(self, timeout: int = 20, proxies: dict = None):
"""
Initialize the SpeechMa TTS client.
-
+
Args:
timeout (int): Request timeout in seconds
proxies (dict): Proxy configuration
@@ -264,7 +266,7 @@ def create_speech(
) -> bytes:
"""
Create speech from text using OpenAI-compatible interface.
-
+
Args:
input (str): The text to convert to speech
voice (str): Voice to use for generation
@@ -273,10 +275,10 @@ def create_speech(
speed (float): Speed of speech (0.25 to 4.0)
instructions (str): Voice instructions (not used by SpeechMa)
**kwargs: Additional parameters (pitch, rate for SpeechMa compatibility)
-
+
Returns:
bytes: Audio data
-
+
Raises:
ValueError: If input parameters are invalid
exceptions.FailedToGenerateResponseError: If generation fails
@@ -286,21 +288,21 @@ def create_speech(
raise ValueError("Input text must be a non-empty string")
if len(input) > 10000:
raise ValueError("Input text exceeds maximum allowed length of 10,000 characters")
-
+
model = self.validate_model(model or self.default_model)
voice = self.validate_voice(voice)
response_format = self.validate_format(response_format)
-
+
# Convert speed to SpeechMa rate parameter
rate = int((speed - 1.0) * 10) # Convert 0.25-4.0 to -7.5 to 30, clamp to -10 to 10
rate = max(-10, min(10, rate))
-
+
# Extract SpeechMa-specific parameters
pitch = kwargs.get('pitch', 0)
-
+
# Map voice to SpeechMa format
speechma_voice = self.voice_mapping.get(voice, self.all_voices.get(voice.title(), "voice-116"))
-
+
# Prepare payload
payload = {
"text": input,
@@ -309,7 +311,7 @@ def create_speech(
"rate": rate,
"volume": 100
}
-
+
try:
response = self.session.post(
self.api_url,
@@ -318,40 +320,40 @@ def create_speech(
timeout=self.timeout
)
response.raise_for_status()
-
+
# Validate audio response
content_type = response.headers.get('content-type', '').lower()
- if ('audio' in content_type or
- response.content.startswith(b'\xff\xfb') or
- response.content.startswith(b'ID3') or
+ if ('audio' in content_type or
+ response.content.startswith(b'\xff\xfb') or
+ response.content.startswith(b'ID3') or
b'LAME' in response.content[:100]):
return response.content
else:
raise exceptions.FailedToGenerateResponseError(
f"Unexpected response format. Content-Type: {content_type}"
)
-
+
except requests.exceptions.RequestException as e:
raise exceptions.FailedToGenerateResponseError(f"API request failed: {e}")
def with_streaming_response(self):
"""
Return a context manager for streaming responses.
-
+
Returns:
SpeechMaStreamingResponse: Context manager for streaming
"""
return SpeechMaStreamingResponse(self)
def tts(
- self,
- text: str,
+ self,
+ text: str,
model: str = None,
- voice: str = "emma",
+ voice: str = "emma",
response_format: str = "mp3",
instructions: str = None,
- pitch: int = 0,
- rate: int = 0,
+ pitch: int = 0,
+ rate: int = 0,
verbose: bool = True
) -> str:
"""
@@ -379,18 +381,18 @@ def tts(
raise ValueError("Input text must be a non-empty string")
if len(text) > 10000:
raise ValueError("Input text exceeds maximum allowed length of 10,000 characters")
-
+
# Validate model, voice, and format using base class methods
model = self.validate_model(model or self.default_model)
voice = self.validate_voice(voice)
response_format = self.validate_format(response_format)
-
+
# Map voice to SpeechMa API format
speechma_voice = self.voice_mapping.get(voice, voice)
if speechma_voice not in self.all_voices.values():
# Fallback to legacy voice mapping
speechma_voice = self.all_voices.get(voice.title(), self.all_voices.get("Emma", "voice-116"))
-
+
# Create temporary file with appropriate extension
file_extension = f".{response_format}" if response_format != "pcm" else ".wav"
filename = pathlib.Path(tempfile.mktemp(suffix=file_extension, dir=self.temp_dir))
@@ -398,10 +400,14 @@ def tts(
# Split text into sentences using the utils module for better processing
sentences = utils.split_sentences(text)
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Processing {len(sentences)} sentences")
- ic.configureOutput(prefix='DEBUG| '); ic(f"Model: {model}")
- ic.configureOutput(prefix='DEBUG| '); ic(f"Voice: {voice} -> {speechma_voice}")
- ic.configureOutput(prefix='DEBUG| '); ic(f"Format: {response_format}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Processing {len(sentences)} sentences")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Model: {model}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Voice: {voice} -> {speechma_voice}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Format: {response_format}")
def generate_audio_for_chunk(part_text: str, part_number: int):
"""
@@ -441,12 +447,13 @@ def generate_audio_for_chunk(part_text: str, part_number: int):
# Check if response is audio data
content_type = response.headers.get('content-type', '').lower()
- if ('audio' in content_type or
- response.content.startswith(b'\xff\xfb') or
- response.content.startswith(b'ID3') or
+ if ('audio' in content_type or
+ response.content.startswith(b'\xff\xfb') or
+ response.content.startswith(b'ID3') or
b'LAME' in response.content[:100]):
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Chunk {part_number} processed successfully")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Chunk {part_number} processed successfully")
return part_number, response.content
else:
raise exceptions.FailedToGenerateResponseError(
@@ -460,7 +467,8 @@ def generate_audio_for_chunk(part_text: str, part_number: int):
f"Failed to generate audio for chunk {part_number} after {max_retries} retries: {e}"
)
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Retrying chunk {part_number} (attempt {retry_count + 1})")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Retrying chunk {part_number} (attempt {retry_count + 1})")
time.sleep(1) # Brief delay before retry
# Process chunks concurrently for better performance
@@ -471,14 +479,15 @@ def generate_audio_for_chunk(part_text: str, part_number: int):
executor.submit(generate_audio_for_chunk, sentence, i): i
for i, sentence in enumerate(sentences)
}
-
+
for future in as_completed(future_to_chunk):
try:
chunk_number, audio_data = future.result()
audio_chunks.append((chunk_number, audio_data))
except Exception as e:
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Error processing chunk: {e}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Error processing chunk: {e}")
raise
else:
# Single sentence, process directly
@@ -494,7 +503,8 @@ def generate_audio_for_chunk(part_text: str, part_number: int):
with open(filename, 'wb') as f:
f.write(combined_audio)
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Audio saved to: {filename}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Audio saved to: {filename}")
return filename.as_posix()
except IOError as e:
raise exceptions.FailedToGenerateResponseError(f"Failed to save audio file: {e}")
@@ -502,16 +512,16 @@ def generate_audio_for_chunk(part_text: str, part_number: int):
class SpeechMaStreamingResponse:
"""Context manager for streaming SpeechMa TTS responses."""
-
+
def __init__(self, client: SpeechMaTTS):
self.client = client
-
+
def __enter__(self):
return self
-
+
def __exit__(self, exc_type, exc_val, exc_tb):
pass
-
+
def create_speech(
self,
input: str,
@@ -524,10 +534,10 @@ def create_speech(
):
"""
Create speech with streaming response simulation.
-
+
Note: SpeechMa doesn't support true streaming, so this returns
the complete audio data wrapped in a BytesIO object.
-
+
Args:
input (str): Text to convert to speech
voice (str): Voice to use
@@ -536,7 +546,7 @@ def create_speech(
speed (float): Speech speed
instructions (str): Voice instructions
**kwargs: Additional parameters
-
+
Returns:
BytesIO: Audio data stream
"""
@@ -556,7 +566,7 @@ def create_speech(
if __name__ == "__main__":
# Initialize the SpeechMa TTS client
speechma = SpeechMaTTS()
-
+
# Example 1: Basic usage with legacy method
print("=== Example 1: Basic TTS ===")
text = "Hello, this is a test of the SpeechMa text-to-speech API."
@@ -565,7 +575,7 @@ def create_speech(
print(f"Audio saved to: {audio_file}")
except Exception as e:
print(f"Error: {e}")
-
+
# Example 2: OpenAI-compatible interface
print("\n=== Example 2: OpenAI-compatible interface ===")
try:
@@ -577,14 +587,14 @@ def create_speech(
speed=1.2
)
print(f"Generated {len(audio_data)} bytes of audio data")
-
+
# Save to file
with open("openai_compatible_test.mp3", "wb") as f:
f.write(audio_data)
print("Audio saved to: openai_compatible_test.mp3")
except Exception as e:
print(f"Error: {e}")
-
+
# Example 3: Streaming response context manager
print("\n=== Example 3: Streaming response ===")
try:
@@ -598,14 +608,14 @@ def create_speech(
print(f"Streamed {len(audio_data)} bytes of audio data")
except Exception as e:
print(f"Error: {e}")
-
+
# Example 4: Voice and model validation
print("\n=== Example 4: Parameter validation ===")
try:
# Test supported voices
print("Supported voices:", speechma.SUPPORTED_VOICES[:5], "...")
print("Supported models:", speechma.SUPPORTED_MODELS)
-
+
# Test with different parameters
audio_file = speechma.tts(
text="Testing different voice parameters.",
@@ -617,4 +627,4 @@ def create_speech(
)
print(f"Audio with custom parameters saved to: {audio_file}")
except Exception as e:
- print(f"Error: {e}")
\ No newline at end of file
+ print(f"Error: {e}")
diff --git a/webscout/Provider/TTS/streamElements.py b/webscout/Provider/TTS/streamElements.py
index e4e55ca9..8341e914 100644
--- a/webscout/Provider/TTS/streamElements.py
+++ b/webscout/Provider/TTS/streamElements.py
@@ -1,16 +1,20 @@
-import time
-import requests
import pathlib
-import urllib.parse
import tempfile
-from typing import Union
+import time
+import urllib.parse
+from concurrent.futures import ThreadPoolExecutor, as_completed
from io import BytesIO
+
+import requests
+from litprinter import ic
+
from webscout import exceptions
from webscout.litagent import LitAgent
-from concurrent.futures import ThreadPoolExecutor, as_completed
+
from . import utils
from .base import BaseTTSProvider
+
class StreamElements(BaseTTSProvider):
"""
@@ -21,7 +25,7 @@ class StreamElements(BaseTTSProvider):
required_auth = False
-
+
# Supported voices
@@ -171,14 +175,17 @@ def generate_audio_for_chunk(part_text: str, part_number: int):
# Check if the request was successful
if response.ok and response.status_code == 200:
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Chunk {part_number} processed successfully")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Chunk {part_number} processed successfully")
return part_number, response.content
else:
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"No data received for chunk {part_number}. Retrying...")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"No data received for chunk {part_number}. Retrying...")
except requests.RequestException as e:
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Error for chunk {part_number}: {e}. Retrying...")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Error for chunk {part_number}: {e}. Retrying...")
time.sleep(1)
try:
# Using ThreadPoolExecutor to handle requests concurrently
@@ -196,25 +203,29 @@ def generate_audio_for_chunk(part_text: str, part_number: int):
audio_chunks[part_number] = audio_data # Store the audio data in correct sequence
except Exception as e:
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Failed to generate audio for chunk {chunk_num}: {e}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Failed to generate audio for chunk {chunk_num}: {e}")
# Combine audio chunks in the correct sequence
combined_audio = BytesIO()
for part_number in sorted(audio_chunks.keys()):
combined_audio.write(audio_chunks[part_number])
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Added chunk {part_number} to the combined file.")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Added chunk {part_number} to the combined file.")
# Save the combined audio data to a single file
with open(filename, 'wb') as f:
f.write(combined_audio.getvalue())
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Final Audio Saved as {filename}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Final Audio Saved as {filename}")
return filename.as_posix()
except requests.exceptions.RequestException as e:
if verbose:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Failed to perform the operation: {e}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Failed to perform the operation: {e}")
raise exceptions.FailedToGenerateResponseError(
f"Failed to perform the operation: {e}"
)
@@ -224,6 +235,8 @@ def generate_audio_for_chunk(part_text: str, part_number: int):
streamelements = StreamElements()
text = "This is a test of the StreamElements text-to-speech API. It supports multiple sentences and advanced logging."
- ic.configureOutput(prefix='DEBUG| '); ic("Generating audio...")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic("Generating audio...")
audio_file = streamelements.tts(text, voice="Mathieu")
- ic.configureOutput(prefix='INFO| '); ic(f"Audio saved to: {audio_file}")
+ ic.configureOutput(prefix='INFO| ')
+ ic(f"Audio saved to: {audio_file}")
diff --git a/webscout/Provider/TTS/utils.py b/webscout/Provider/TTS/utils.py
index db7acec0..47735c61 100644
--- a/webscout/Provider/TTS/utils.py
+++ b/webscout/Provider/TTS/utils.py
@@ -1,13 +1,13 @@
"""
Text processing utilities for TTS providers.
"""
-from typing import Union, List, Dict, Tuple, Set, Optional, Pattern
import re
+from typing import Dict, List, Pattern, Set, Tuple
class SentenceTokenizer:
"""Advanced sentence tokenizer with support for complex cases and proper formatting."""
-
+
def __init__(self) -> None:
# Common abbreviations by category
self.TITLES: Set[str] = {
@@ -15,31 +15,31 @@ def __init__(self) -> None:
'hon', 'pres', 'gov', 'atty', 'supt', 'det', 'rev', 'col','maj', 'gen', 'capt', 'cmdr',
'lt', 'sgt', 'cpl', 'pvt'
}
-
+
self.ACADEMIC: Set[str] = {
'ph.d', 'phd', 'm.d', 'md', 'b.a', 'ba', 'm.a', 'ma', 'd.d.s', 'dds',
'm.b.a', 'mba', 'b.sc', 'bsc', 'm.sc', 'msc', 'llb', 'll.b', 'bl'
}
-
+
self.ORGANIZATIONS: Set[str] = {
'inc', 'ltd', 'co', 'corp', 'llc', 'llp', 'assn', 'bros', 'plc', 'cos',
'intl', 'dept', 'est', 'dist', 'mfg', 'div'
}
-
+
self.MONTHS: Set[str] = {
'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'
}
-
+
self.UNITS: Set[str] = {
'oz', 'pt', 'qt', 'gal', 'ml', 'cc', 'km', 'cm', 'mm', 'ft', 'in',
'kg', 'lb', 'lbs', 'hz', 'khz', 'mhz', 'ghz', 'kb', 'mb', 'gb', 'tb'
}
-
+
self.TECHNOLOGY: Set[str] = {
'v', 'ver', 'app', 'sys', 'dir', 'exe', 'lib', 'api', 'sdk', 'url',
'cpu', 'gpu', 'ram', 'rom', 'hdd', 'ssd', 'lan', 'wan', 'sql', 'html'
}
-
+
self.MISC: Set[str] = {
'vs', 'etc', 'ie', 'eg', 'no', 'al', 'ca', 'cf', 'pp', 'est', 'st',
'approx', 'appt', 'apt', 'dept', 'depts', 'min', 'max', 'avg'
@@ -60,13 +60,13 @@ def __init__(self) -> None:
self.NUMBER_PATTERN: str = (
r'\d+(?:\.\d+)?(?:%|°|km|cm|mm|m|kg|g|lb|ft|in|mph|kmh|hz|mhz|ghz)?'
)
-
+
# Quote and bracket pairs
self.QUOTE_PAIRS: Dict[str, str] = {
- '"': '"', "'": "'", '"': '"', "「": "」", "『": "』",
- "«": "»", "‹": "›", "'": "'", "‚": "'"
+ '"': '"', "'": "'", "「": "」", "『": "』",
+ "«": "»", "‹": "›", "‚": "'"
}
-
+
self.BRACKETS: Dict[str, str] = {
'(': ')', '[': ']', '{': '}', '⟨': '⟩', '「': '」',
'『': '』', '【': '】', '〖': '〗', '「': '」'
@@ -84,14 +84,14 @@ def _compile_patterns(self) -> None:
(?:
# Standard endings with optional quotes/brackets
(?<=[.!?])[\"\'\)\]\}»›」』\s]*
-
+
# Ellipsis
|(?:\.{2,}|…)
-
+
# Asian-style endings
|(?<=[。!?」』】\s])
)
-
+
# Must be followed by whitespace and capital letter or number
(?=\s+(?:[A-Z0-9]|["'({[\[「『《‹〈][A-Z]))
''',
@@ -169,30 +169,30 @@ def _restore_formatting(self, sentences: List[str]) -> List[str]:
for sentence in sentences:
# Restore dots in abbreviations
sentence = sentence.replace('__DOT__', '.')
-
+
# Restore paragraph breaks
sentence = sentence.replace('__PARA__', '\n\n')
-
+
# Clean up whitespace
sentence = re.sub(r'\s+', ' ', sentence).strip()
-
+
# Capitalize first letter if it's lowercase and not an abbreviation
words = sentence.split()
if words and words[0].lower() not in self.all_abbreviations:
sentence = sentence[0].upper() + sentence[1:]
-
+
if sentence:
restored.append(sentence)
-
+
return restored
def tokenize(self, text: str) -> List[str]:
"""
Split text into sentences while handling complex cases.
-
+
Args:
text (str): Input text to split into sentences.
-
+
Returns:
List[str]: List of properly formatted sentences.
"""
@@ -201,31 +201,31 @@ def tokenize(self, text: str) -> List[str]:
# Step 1: Protect special cases
protected_text, placeholders = self._protect_special_cases(text)
-
+
# Step 2: Normalize whitespace
protected_text = self._normalize_whitespace(protected_text)
-
+
# Step 3: Handle abbreviations
protected_text = self._handle_abbreviations(protected_text)
-
+
# Step 4: Split into potential sentences
potential_sentences = self.SENTENCE_END.split(protected_text)
-
+
# Step 5: Process and restore formatting
sentences = self._restore_formatting(potential_sentences)
-
+
# Step 6: Restore special cases
sentences = [self._restore_special_cases(s, placeholders) for s in sentences]
-
+
# Step 7: Post-process sentences
final_sentences = []
current_sentence = []
-
+
for sentence in sentences:
# Skip empty sentences
if not sentence.strip():
continue
-
+
# Check if sentence might be continuation of previous
if current_sentence and sentence[0].islower():
current_sentence.append(sentence)
@@ -233,21 +233,21 @@ def tokenize(self, text: str) -> List[str]:
if current_sentence:
final_sentences.append(' '.join(current_sentence))
current_sentence = [sentence]
-
+
# Add last sentence if exists
if current_sentence:
final_sentences.append(' '.join(current_sentence))
-
+
return final_sentences
def split_sentences(text: str) -> List[str]:
"""
Convenience function to split text into sentences using SentenceTokenizer.
-
+
Args:
text (str): Input text to split into sentences.
-
+
Returns:
List[str]: List of properly formatted sentences.
"""
@@ -258,19 +258,19 @@ def split_sentences(text: str) -> List[str]:
if __name__ == "__main__":
# Test text with various challenging cases
test_text: str = """
- Dr. Smith (Ph.D., M.D.) visited Washington D.C. on Jan. 20, 2024! He met with Prof. Johnson at 3:30 p.m.
- They discussed A.I. and machine learning... "What about the U.S. market?" asked Dr. Smith.
+ Dr. Smith (Ph.D., M.D.) visited Washington D.C. on Jan. 20, 2024! He met with Prof. Johnson at 3:30 p.m.
+ They discussed A.I. and machine learning... "What about the U.S. market?" asked Dr. Smith.
The meeting ended at 5 p.m. Later, they went to Mr. Wilson's house (located at 123 Main St.) for dinner.
-
+
Visit our website at https://www.example.com or email us at test@example.com!
The temperature was 72.5°F (22.5°C). The company's Q3 2023 revenue was $12.5M USD.
-
+
「これは日本語の文章です。」This is a mixed-language text! How cool is that?
-
+
Some technical specs: CPU: 3.5GHz, RAM: 16GB, Storage: 2TB SSD.
Common abbreviations: etc., i.e., e.g., vs., cf., approx. 100 units.
"""
-
+
# Process and print each sentence
sentences: List[str] = split_sentences(test_text)
print("Detected sentences:")
diff --git a/webscout/Provider/TextPollinationsAI.py b/webscout/Provider/TextPollinationsAI.py
index 9c45cdc0..8661512e 100644
--- a/webscout/Provider/TextPollinationsAI.py
+++ b/webscout/Provider/TextPollinationsAI.py
@@ -1,13 +1,15 @@
-import requests
import json
-from typing import Union, Any, Dict, Generator, Optional, List
+from typing import Any, Dict, Generator, List, Optional, Union
+
+import requests
-from webscout.AIutel import Optimizers, Conversation, AwesomePrompts, sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
from webscout.litagent import LitAgent as Lit
+
class TextPollinationsAI(Provider):
"""
A class to interact with the Pollinations AI API.
@@ -15,7 +17,7 @@ class TextPollinationsAI(Provider):
required_auth = False
_models_url = "https://text.pollinations.ai/models"
-
+
# Static list as fallback
AVAILABLE_MODELS = [
"deepseek",
@@ -39,14 +41,14 @@ class TextPollinationsAI(Provider):
def __init__(self,
is_conversation: bool = True,
- max_tokens: int = 8096,
+ max_tokens: int = 8096,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
model: str = "openai",
system_prompt: str = "You are a helpful AI assistant.",
):
@@ -63,7 +65,7 @@ def __init__(self,
# Fetch latest models dynamically to ensure we have the most up-to-date list
self.update_available_models()
-
+
if model not in self.AVAILABLE_MODELS:
# warn or just allow it? allowing it for flexibility
if model not in self.AVAILABLE_MODELS:
@@ -119,11 +121,12 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
tools: Optional[List[Dict[str, Any]]] = None,
tool_choice: Optional[Dict[str, Any]] = None,
- ) -> Union[Dict[str, Any], Generator[Any, None, None]]:
+ **kwargs: Any,
+ ) -> Response:
"""Chat with AI"""
conversation_prompt = self.conversation.gen_complete_prompt(prompt)
if optimizer:
@@ -157,7 +160,7 @@ def for_stream():
timeout=self.timeout
)
response.raise_for_status()
-
+
streaming_text = ""
processed_stream = sanitize_stream(
data=response.iter_content(chunk_size=None),
@@ -190,7 +193,7 @@ def for_stream():
self.last_response.update(dict(text=streaming_text))
if streaming_text:
self.conversation.update_chat_history(prompt, streaming_text)
-
+
except Exception as e:
raise exceptions.FailedToGenerateResponseError(f"Stream request failed: {e}") from e
@@ -204,7 +207,7 @@ def for_non_stream():
timeout=self.timeout
)
response.raise_for_status()
-
+
# Use sanitize_stream to parse the non-streaming JSON response
processed_stream = sanitize_stream(
data=response.text,
@@ -215,19 +218,23 @@ def for_non_stream():
)
# Extract the single result
resp_json = next(processed_stream, None)
-
+
# Check for standard OpenAI response structure
if resp_json and 'choices' in resp_json and len(resp_json['choices']) > 0:
-
+ choice = resp_json['choices'][0]
+ content = choice.get('message', {}).get('content')
+ tool_calls = choice.get('message', {}).get('tool_calls')
+ result = content if content else (tool_calls if tool_calls else "")
+
self.last_response = result
self.conversation.update_chat_history(prompt, content or "")
-
+
if raw:
return content if content else (json.dumps(tool_calls) if tool_calls else "")
return result
-
+
else:
- return {}
+ return {}
except Exception as e:
raise exceptions.FailedToGenerateResponseError(f"Non-stream request failed: {e}") from e
@@ -238,7 +245,7 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
tools: Optional[List[Dict[str, Any]]] = None,
tool_choice: Optional[Dict[str, Any]] = None,
@@ -281,25 +288,25 @@ def get_message(self, response: dict) -> str:
print("-" * 80)
print(f"{'Model':<50} {'Status':<10} {'Response'}")
print("-" * 80)
-
+
# Test only a subset to be fast
- test_models = ["openai", "gemini"]
-
+ test_models = ["openai", "gemini"]
+
for model in test_models:
try:
print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True)
test_ai = TextPollinationsAI(model=model, timeout=60)
-
+
# Non-stream test
start_response = test_ai.chat("Hello!", stream=False)
- if start_response:
+ if start_response and isinstance(start_response, str):
status = "✓"
display = start_response[:30] + "..."
else:
status = "✗"
- display = "Empty"
-
+ display = "Empty or invalid type"
+
print(f"\r{model:<50} {status:<10} {display}")
-
+
except Exception as e:
- print(f"\r{model:<50} {'✗':<10} {str(e)[:50]}")
\ No newline at end of file
+ print(f"\r{model:<50} {'✗':<10} {str(e)[:50]}")
diff --git a/webscout/Provider/TogetherAI.py b/webscout/Provider/TogetherAI.py
index 9d6db3ba..3dc21783 100644
--- a/webscout/Provider/TogetherAI.py
+++ b/webscout/Provider/TogetherAI.py
@@ -31,25 +31,25 @@ def get_models(cls, api_key: str = None):
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json"
}
-
+
response = session.get(
"https://api.together.xyz/v1/models",
headers=headers,
impersonate="chrome110"
)
-
+
if response.status_code != 200:
return cls.AVAILABLE_MODELS
-
+
data = response.json()
# Together API returns a list of model objects
if isinstance(data, list):
# Filter for chat/language models if possible, or just return all IDs
# The API returns objects with 'id', 'type', etc.
return [model["id"] for model in data if isinstance(model, dict) and "id" in model]
-
+
return cls.AVAILABLE_MODELS
-
+
except Exception:
return cls.AVAILABLE_MODELS
diff --git a/webscout/Provider/TwoAI.py b/webscout/Provider/TwoAI.py
index faed86bd..9e8b681c 100644
--- a/webscout/Provider/TwoAI.py
+++ b/webscout/Provider/TwoAI.py
@@ -1,16 +1,13 @@
-from curl_cffi.requests import Session
-from curl_cffi import CurlError
-import json
import base64
-from typing import Any, Dict, Optional, Generator, Union
-import re # Import re for parsing SSE
+import json
+from typing import Any, Dict, Generator, Optional, Union
+
+from curl_cffi import CurlError
+from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
-from webscout.litagent import LitAgent
+from webscout.AIbase import Provider
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
class TwoAI(Provider):
@@ -297,7 +294,7 @@ def chat(
aggregated_text += response_dict["text"]
elif isinstance(response_dict, str):
aggregated_text += response_dict
-
+
return aggregated_text
def get_message(self, response: dict) -> str:
@@ -310,4 +307,4 @@ def get_message(self, response: dict) -> str:
ai = TwoAI(api_key="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VySWQiOiJzanl2OHJtZGxDZDFnQ2hQdGxzZHdxUlVteXkyIiwic291cmNlIjoiRmlyZWJhc2UiLCJpYXQiOjE3NTc4NTEyMzYsImV4cCI6MTc1Nzg1MjEzNn0.ilTYrHRdN3_cme6VW3knWWfbypY_n_gsUe9DeDhEwrM", model="sutra-v2", temperature=0.7)
response = ai.chat("Write a poem about AI in the style of Shakespeare.")
for chunk in response:
- print(chunk, end="", flush=True)
\ No newline at end of file
+ print(chunk, end="", flush=True)
diff --git a/webscout/Provider/TypliAI.py b/webscout/Provider/TypliAI.py
index b9a5eb18..b285c891 100644
--- a/webscout/Provider/TypliAI.py
+++ b/webscout/Provider/TypliAI.py
@@ -1,14 +1,15 @@
import random
import string
-from typing import Union, Any, Dict, Generator
+from typing import Any, Dict, Generator, Union
+
from curl_cffi import CurlError
from curl_cffi.requests import Session
-# from curl_cffi.const import CurlHttpVersion # Not strictly needed if using default
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream
-from webscout.AIbase import Provider
+
from webscout import exceptions
+from webscout.AIbase import Provider
+
+# from curl_cffi.const import CurlHttpVersion # Not strictly needed if using default
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
from webscout.litagent import LitAgent
@@ -33,10 +34,10 @@ class TypliAI(Provider):
"""
required_auth = False
AVAILABLE_MODELS = [
- "openai/gpt-4.1-mini",
+ "openai/gpt-4.1-mini",
"openai/gpt-4.1",
- "openai/gpt-5-mini",
- "openai/gpt-5.2",
+ "openai/gpt-5-mini",
+ "openai/gpt-5.2",
"openai/gpt-5.2-pro",
"google/gemini-2.5-flash",
"anthropic/claude-haiku-4-5",
@@ -309,4 +310,4 @@ def get_message(self, response: dict) -> str:
for chunk in response:
print(chunk, end="", flush=True)
except Exception as e:
- print(f"An error occurred: {e}")
\ No newline at end of file
+ print(f"An error occurred: {e}")
diff --git a/webscout/Provider/UNFINISHED/ChatHub.py b/webscout/Provider/UNFINISHED/ChatHub.py
index a387e54f..6db4ddf7 100644
--- a/webscout/Provider/UNFINISHED/ChatHub.py
+++ b/webscout/Provider/UNFINISHED/ChatHub.py
@@ -1,13 +1,12 @@
-import requests
import json
-import os
-from typing import Any, Dict, Optional, Generator, List, Union
+from typing import Any, Dict, Generator, Union
+
+import requests
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
+
class ChatHub(Provider):
"""
@@ -31,7 +30,7 @@ class ChatHub(Provider):
def __init__(
self,
is_conversation: bool = True,
- max_tokens: int = 2049,
+ max_tokens: int = 2049,
timeout: int = 30,
intro: str = None,
filepath: str = None,
@@ -39,7 +38,7 @@ def __init__(
proxies: dict = {},
history_offset: int = 10250,
act: str = None,
- model: str = "sonar-online",
+ model: str = "sonar-online",
):
"""Initializes the ChatHub API client."""
self.url = "https://app.chathub.gg"
@@ -53,8 +52,8 @@ def __init__(
'X-App-Id': 'web'
}
self.session = requests.Session()
- self.session.headers.update(self.headers)
- self.session.proxies.update(proxies)
+ self.session.headers.update(self.headers)
+ self.session.proxies.update(proxies)
self.timeout = timeout
self.last_response = {}
@@ -127,7 +126,7 @@ def ask(
def for_stream():
try:
with requests.post(self.api_endpoint, headers=self.headers, json=data, stream=True, timeout=self.timeout) as response:
- response.raise_for_status()
+ response.raise_for_status()
streaming_text = ""
# Use sanitize_stream for processing
@@ -214,4 +213,4 @@ def get_message(self, response: dict) -> str:
for chunk in response:
print(chunk, end="", flush=True)
except Exception as e:
- print(f"An error occurred: {e}")
\ No newline at end of file
+ print(f"An error occurred: {e}")
diff --git a/webscout/Provider/UNFINISHED/ChutesAI.py b/webscout/Provider/UNFINISHED/ChutesAI.py
index aaedffec..8c940e79 100644
--- a/webscout/Provider/UNFINISHED/ChutesAI.py
+++ b/webscout/Provider/UNFINISHED/ChutesAI.py
@@ -1,21 +1,29 @@
-import requests
import json
-import time
-import uuid
-from typing import List, Dict, Optional, Union, Generator, Any
-import re
import random
+import re
import string
+import time
+import uuid
+from typing import Any, Dict, Generator, List, Optional, Union
+
+import cloudscraper
+import requests
from rich import print
+
from webscout.litagent.agent import LitAgent
-import cloudscraper
+
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
)
+
# --- ChutesAI API Key Auto-Generator ---
def generate_chutesai_api_key():
url = "https://chutes.ai/auth/start?/create"
@@ -46,10 +54,10 @@ def generate_username(length=8):
scraper = cloudscraper.create_scraper()
response = scraper.post(url, headers=headers, data=data)
print(f"[bold green]Status:[/] {response.status_code}")
-
+
# Ensure response is decoded as UTF-8
response.encoding = 'utf-8'
-
+
try:
resp_json = response.json()
except Exception:
@@ -258,21 +266,21 @@ class ChutesAI(OpenAICompatibleProvider):
def __init__(self, api_key: str = None,):
self.timeout = None # Infinite timeout
self.base_url = "https://llm.chutes.ai/v1/chat/completions"
-
+
# Always generate a new API key, ignore any provided key
print("[yellow]Generating new ChutesAI API key...[/]")
self.api_key = generate_chutesai_api_key()
-
+
if not self.api_key:
print("[red]Failed to generate API key. Retrying...[/]")
# Retry once more
self.api_key = generate_chutesai_api_key()
-
+
if not self.api_key:
raise ValueError("Failed to generate ChutesAI API key after multiple attempts.")
-
+
print(f"[green]Successfully generated API key: {self.api_key[:20]}...[/]")
-
+
self.scraper = cloudscraper.create_scraper()
self.headers = {
"Authorization": f"Bearer {self.api_key}",
@@ -292,12 +300,12 @@ def list(inner_self):
try:
# Example usage - always use generated API key
client = ChutesAI()
-
+
messages = [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "What is the capital of France?"}
]
-
+
print("[cyan]Making API request...[/]")
response = client.chat.completions.create(
model="deepseek-ai/DeepSeek-V3-0324",
@@ -311,7 +319,7 @@ def list(inner_self):
else:
chunk_dict = chunk.dict(exclude_none=True)
print(f"[green]Response Chunk:[/] {chunk_dict}")
-
+
except Exception as e:
print(f"[red]Error: {e}[/]")
- print("[yellow]If the issue persists, the ChutesAI service might be down or the API key generation method needs updating.[/]")
\ No newline at end of file
+ print("[yellow]If the issue persists, the ChutesAI service might be down or the API key generation method needs updating.[/]")
diff --git a/webscout/Provider/UNFINISHED/GizAI.py b/webscout/Provider/UNFINISHED/GizAI.py
index 82098e23..106b2aac 100644
--- a/webscout/Provider/UNFINISHED/GizAI.py
+++ b/webscout/Provider/UNFINISHED/GizAI.py
@@ -1,28 +1,27 @@
-import os
import base64
-import random
import json
-from typing import Union, Dict, Any, Optional, Generator
+import os
+import random
+from typing import Any, Dict, Generator, Union
from urllib import response
from curl_cffi import CurlError
-from curl_cffi.requests import Session
from curl_cffi.const import CurlHttpVersion
+from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers
from webscout.litagent import LitAgent
+
class GizAI(Provider):
"""
A class to interact with the GizAI API.
-
+
Attributes:
system_prompt (str): The system prompt to define the assistant's role.
-
+
Examples:
>>> from webscout.Provider.GizAI import GizAI
>>> ai = GizAI()
@@ -55,7 +54,7 @@ class GizAI(Provider):
"phi-4",
"qwq-32b"
]
-
+
def __init__(
self,
is_conversation: bool = True,
@@ -73,15 +72,15 @@ def __init__(
"""Initializes the GizAI API client."""
if model not in self.AVAILABLE_MODELS:
raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}")
-
+
self.api_url = "https://app.giz.ai/api/data/users/inferenceServer.infer"
-
+
# Initialize LitAgent for user-agent generation
self.agent = LitAgent()
-
+
# Initialize curl_cffi Session
self.session = Session()
-
+
# Set up the headers
self.headers = {
"accept": "application/json, text/plain, */*",
@@ -93,11 +92,11 @@ def __init__(
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin"
}
-
+
# Update session headers and proxies
self.session.headers.update(self.headers)
self.session.proxies = proxies
-
+
# Store configuration
self.system_prompt = system_prompt
self.is_conversation = is_conversation
@@ -105,13 +104,13 @@ def __init__(
self.timeout = timeout
self.last_response = {}
self.model = model
-
+
self.__available_optimizers = (
method
for method in dir(Optimizers)
if callable(getattr(Optimizers, method)) and not method.startswith("__")
)
-
+
Conversation.intro = (
AwesomePrompts().get_act(
act, raise_not_found=True, default=None, case_insensitive=True
@@ -119,22 +118,22 @@ def __init__(
if act
else intro or Conversation.intro
)
-
+
self.conversation = Conversation(
is_conversation, self.max_tokens_to_sample, filepath, update_file
)
self.conversation.history_offset = history_offset
-
+
def _generate_id(self, length: int = 21) -> str:
"""Generates a random URL-safe base64 string."""
random_bytes = os.urandom(length * 2) # Generate more bytes initially
b64_encoded = base64.urlsafe_b64encode(random_bytes).decode('utf-8')
return b64_encoded[:length]
-
+
def _get_random_ip(self) -> str:
"""Generates a random IPv4 address string."""
return f"{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}"
-
+
def ask(
self,
prompt: str,
@@ -145,17 +144,17 @@ def ask(
) -> Dict[str, Any]:
"""
Sends a prompt to the GizAI API and returns the response.
-
+
Args:
prompt (str): The prompt to send to the API.
stream (bool): Not supported by GizAI, kept for compatibility.
raw (bool): Whether to return the raw response.
optimizer (str): Optimizer to use for the prompt.
conversationally (bool): Whether to generate the prompt conversationally.
-
+
Returns:
Dict[str, Any]: The API response.
-
+
Examples:
>>> ai = GizAI()
>>> response = ai.ask("Tell me a joke!")
@@ -168,12 +167,12 @@ def ask(
)
else:
raise Exception(f"Optimizer is not one of {self.__available_optimizers}")
-
+
# Generate random IDs for request
instance_id = self._generate_id()
subscribe_id = self._generate_id()
x_forwarded_for = self._get_random_ip()
-
+
# Set up request body - GizAI doesn't support streaming
request_body = {
"model": "chat",
@@ -189,10 +188,10 @@ def ask(
"instanceId": instance_id,
"subscribeId": subscribe_id
}
-
+
# Combine default headers with the dynamic x-forwarded-for header
request_headers = {**self.headers, "x-forwarded-for": x_forwarded_for}
-
+
try:
# Use curl_cffi session post with impersonate
response = self.session.post(
@@ -204,7 +203,7 @@ def ask(
http_version=CurlHttpVersion.V2_0 # Use HTTP/2
)
response.raise_for_status() # Check for HTTP errors
-
+
# Process the response
try:
response_json = response.json()
@@ -221,19 +220,19 @@ def ask(
except json.JSONDecodeError:
# Handle case where response is not valid JSON
content = response.text
-
+
# Update conversation history
self.last_response = {"text": content}
self.conversation.update_chat_history(prompt, content)
-
+
return self.last_response if not raw else content
-
+
except CurlError as e:
raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {str(e)}")
except Exception as e:
error_text = getattr(e, 'response', None) and getattr(e.response, 'text', '')
raise exceptions.FailedToGenerateResponseError(f"Request failed ({type(e).__name__}): {str(e)} - {error_text}")
-
+
def chat(
self,
prompt: str,
@@ -243,16 +242,16 @@ def chat(
) -> 'Generator[str, None, None]':
"""
Generates a response from the GizAI API.
-
+
Args:
prompt (str): The prompt to send to the API.
stream (bool): Not supported by GizAI, kept for compatibility.
optimizer (str): Optimizer to use for the prompt.
conversationally (bool): Whether to generate the prompt conversationally.
-
+
Returns:
Generator[str, None, None]: The API response text as a generator.
-
+
Examples:
>>> ai = GizAI()
>>> response = ai.chat("What's the weather today?")
@@ -267,17 +266,17 @@ def chat(
yield result
else:
return result
-
+
def get_message(self, response: Union[dict, str]) -> str:
"""
Extracts the message from the API response.
-
+
Args:
response (Union[dict, str]): The API response.
-
+
Returns:
str: The message content.
-
+
Examples:
>>> ai = GizAI()
>>> response = ai.ask("Tell me a joke!")
@@ -292,4 +291,4 @@ def get_message(self, response: Union[dict, str]) -> str:
ai = GizAI()
response = ai.chat("Hello, how are you?", stream=True)
for chunk in response:
- print(chunk, end="", flush=True)
\ No newline at end of file
+ print(chunk, end="", flush=True)
diff --git a/webscout/Provider/UNFINISHED/Marcus.py b/webscout/Provider/UNFINISHED/Marcus.py
index 7e8a6729..a2d390e2 100644
--- a/webscout/Provider/UNFINISHED/Marcus.py
+++ b/webscout/Provider/UNFINISHED/Marcus.py
@@ -1,13 +1,17 @@
-from curl_cffi.requests import Session
+from typing import Any, Dict, Generator, Union
+
from curl_cffi import CurlError
-import json
-from typing import Union, Any, Dict, Optional, Generator
+from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider
+from webscout.AIutel import ( # Import sanitize_stream
+ AwesomePrompts,
+ Conversation,
+ Optimizers,
+ sanitize_stream,
+)
+
class Marcus(Provider):
"""
@@ -35,14 +39,14 @@ def __init__(
self.api_endpoint = "https://www.askmarcus.app/api/response"
self.timeout = timeout
self.last_response = {}
-
+
self.headers = {
'content-type': 'application/json',
'accept': '*/*',
'origin': 'https://www.askmarcus.app',
'referer': 'https://www.askmarcus.app/chat',
}
-
+
# Update curl_cffi session headers and proxies
self.session.headers.update(self.headers)
self.session.proxies = proxies # Assign proxies directly
@@ -102,7 +106,7 @@ def for_stream():
impersonate="chrome110" # Use a common impersonation profile
)
response.raise_for_status() # Check for HTTP errors
-
+
# Use sanitize_stream to decode bytes and yield text chunks
processed_stream = sanitize_stream(
data=response.iter_content(chunk_size=None), # Pass byte iterator
@@ -139,7 +143,7 @@ def for_non_stream():
impersonate="chrome110" # Use a common impersonation profile
)
response.raise_for_status() # Check for HTTP errors
-
+
response_text_raw = response.text # Get raw text
# Process the text using sanitize_stream (even though it's not streaming)
diff --git a/webscout/Provider/UNFINISHED/Qodo.py b/webscout/Provider/UNFINISHED/Qodo.py
index 456bbbbb..92eb7312 100644
--- a/webscout/Provider/UNFINISHED/Qodo.py
+++ b/webscout/Provider/UNFINISHED/Qodo.py
@@ -1,16 +1,15 @@
-from curl_cffi.requests import Session
-from curl_cffi import CurlError
-from typing import Any, Dict, Optional, Generator, Union
import uuid
-import json
+from typing import Any, Dict, Generator, Optional, Union
+
+from curl_cffi import CurlError
+from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
from webscout.litagent import LitAgent
+
class QodoAI(Provider):
"""
A class to interact with the Qodo AI API.
@@ -18,7 +17,7 @@ class QodoAI(Provider):
AVAILABLE_MODELS = [
"gpt-4.1",
- "gpt-4o",
+ "gpt-4o",
"o3",
"o4-mini",
"claude-4-sonnet",
@@ -56,21 +55,21 @@ def __init__(
"""Initializes the Qodo AI API client."""
if model not in self.AVAILABLE_MODELS:
raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}")
-
+
self.url = "https://api.cli.qodo.ai/v2/agentic/start-task"
self.info_url = "https://api.cli.qodo.ai/v2/info/get-things"
-
+
# Initialize LitAgent for user agent generation
self.agent = LitAgent()
self.fingerprint = self.agent.generate_fingerprint(browser)
-
+
# Store API key
self.api_key = api_key or "sk-dS7U-extxMWUxc8SbYYOuncqGUIE8-y2OY8oMCpu0eI-qnSUyH9CYWO_eAMpqwfMo7pXU3QNrclfZYMO0M6BJTM"
-
+
# Generate session ID dynamically from API
self.session_id = self._get_session_id()
self.request_id = str(uuid.uuid4())
-
+
# Use the fingerprint for headers
self.headers = {
"Accept": "text/plain",
@@ -83,7 +82,7 @@ def __init__(
"Request-id": self.request_id,
"User-Agent": self.fingerprint["user_agent"],
}
-
+
# Initialize curl_cffi Session
self.session = Session()
# Add Session-id to headers after getting it from API
@@ -118,23 +117,23 @@ def __init__(
def refresh_identity(self, browser: str = None):
"""
Refreshes the browser identity fingerprint.
-
+
Args:
browser: Specific browser to use for the new fingerprint
"""
browser = browser or self.fingerprint.get("browser_type", "chrome")
self.fingerprint = self.agent.generate_fingerprint(browser)
-
+
# Update headers with new fingerprint
self.headers.update({
"Accept-Language": self.fingerprint["accept_language"],
"User-Agent": self.fingerprint["user_agent"],
})
-
+
# Update session headers
for header, value in self.headers.items():
self.session.headers[header] = value
-
+
return self.fingerprint
def _build_payload(self, prompt: str):
@@ -379,13 +378,13 @@ def _get_session_id(self) -> str:
"User-Agent": self.fingerprint["user_agent"] if hasattr(self, 'fingerprint') else "axios/1.10.0",
}
temp_session.headers.update(temp_headers)
-
+
response = temp_session.get(
self.info_url,
timeout=self.timeout if hasattr(self, 'timeout') else 30,
impersonate="chrome110"
)
-
+
if response.status_code == 200:
data = response.json()
session_id = data.get("session-id")
@@ -406,12 +405,12 @@ def _get_session_id(self) -> str:
"Usage: QodoAI(api_key='your_api_key_here')\n"
"To get an API key, install Qodo CLI via: https://docs.qodo.ai/qodo-documentation/qodo-gen-cli/getting-started/setup-and-quickstart"
)
-
+
# Fallback to generated session ID if API call fails
from datetime import datetime
today = datetime.now().strftime("%Y%m%d")
return f"{today}-{str(uuid.uuid4())}"
-
+
except exceptions.FailedToGenerateResponseError:
# Re-raise our custom exceptions
raise
@@ -427,23 +426,22 @@ def _get_session_id(self) -> str:
def refresh_session(self):
"""
Refreshes the session ID by calling the Qodo API.
-
+
Returns:
str: The new session ID
"""
- old_session_id = self.session_id
self.session_id = self._get_session_id()
-
+
# Update headers with new session ID
self.headers["Session-id"] = self.session_id
self.session.headers["Session-id"] = self.session_id
-
+
return self.session_id
def get_available_models(self) -> Dict[str, Any]:
"""
Get available models and info from Qodo API.
-
+
Returns:
Dict containing models, default_model, version, and session info
"""
@@ -453,7 +451,7 @@ def get_available_models(self) -> Dict[str, Any]:
timeout=self.timeout,
impersonate=self.fingerprint.get("browser_type", "chrome110")
)
-
+
if response.status_code == 200:
return response.json()
elif response.status_code == 401:
@@ -464,7 +462,7 @@ def get_available_models(self) -> Dict[str, Any]:
)
else:
raise exceptions.FailedToGenerateResponseError(f"Failed to get models: HTTP {response.status_code}")
-
+
except CurlError as e:
raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {e}")
except Exception as e:
@@ -473,6 +471,6 @@ def get_available_models(self) -> Dict[str, Any]:
if __name__ == "__main__":
ai = QodoAI() # u will need to give your API key here to get api install qodo cli via https://docs.qodo.ai/qodo-documentation/qodo-gen-cli/getting-started/setup-and-quickstart
- response = ai.chat("write a poem about india", raw=False, stream=True)
+ response = ai.chat("write a poem about india", raw=False, stream=True)
for chunk in response:
- print(chunk, end='', flush=True)
\ No newline at end of file
+ print(chunk, end='', flush=True)
diff --git a/webscout/Provider/UNFINISHED/XenAI.py b/webscout/Provider/UNFINISHED/XenAI.py
index 87a3192d..dc924821 100644
--- a/webscout/Provider/UNFINISHED/XenAI.py
+++ b/webscout/Provider/UNFINISHED/XenAI.py
@@ -1,17 +1,15 @@
-import json
-import uuid
import random
import string
+import uuid
+import warnings
from typing import Any, Dict, Generator, Union
+
import requests
-import warnings
import urllib3
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
from webscout.litagent import LitAgent
# Suppress only the single InsecureRequestWarning from urllib3 needed for verify=False
@@ -69,7 +67,7 @@ def __init__(
print(f"Warning: Model '{model}' is not listed in AVAILABLE_MODELS. Proceeding with the provided model.")
self.api_endpoint = "https://chat.xenai.tech/api/chat/completions"
-
+
self.model = model
self.system_prompt = system_prompt
diff --git a/webscout/Provider/UNFINISHED/Youchat.py b/webscout/Provider/UNFINISHED/Youchat.py
index 349decd5..81f25a9d 100644
--- a/webscout/Provider/UNFINISHED/Youchat.py
+++ b/webscout/Provider/UNFINISHED/Youchat.py
@@ -1,14 +1,14 @@
-from uuid import uuid4
-import json
import datetime
-from typing import Union, Dict, Any, Generator
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream
-from webscout.AIbase import Provider
-from webscout import exceptions
+import json
+from typing import Generator, Union
+from uuid import uuid4
+
import cloudscraper
+from webscout import exceptions
+from webscout.AIbase import Provider
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers
+
class YouChat(Provider):
"""
@@ -34,7 +34,7 @@ class YouChat(Provider):
# "deepseek_r1", # isProOnly: true
# "deepseek_v3", # isProOnly: true
# "gemini_2_5_pro_experimental", # isProOnly: true
-
+
# Free models (isProOnly: false)
"gpt_4o_mini",
"gpt_4o",
@@ -52,7 +52,7 @@ class YouChat(Provider):
"llama3_1_405b",
"mistral_large_2",
"command_r_plus",
-
+
# Free models not enabled for user chat modes
"llama3_3_70b", # isAllowedForUserChatModes: false
"llama3_2_90b", # isAllowedForUserChatModes: false
@@ -174,10 +174,10 @@ def ask(
trace_id = str(uuid4())
conversation_turn_id = str(uuid4())
-
+
# Current timestamp in ISO format for traceId
current_time = datetime.datetime.now().isoformat()
-
+
# Updated query parameters to match the new API format
params = {
"page": 1,
@@ -197,7 +197,7 @@ def ask(
"traceId": f"{trace_id}|{conversation_turn_id}|{current_time}",
"use_nested_youchat_updates": "true"
}
-
+
# New payload format is JSON
payload = {
"query": conversation_prompt,
@@ -206,12 +206,12 @@ def ask(
def for_stream():
response = self.session.post(
- self.chat_endpoint,
- headers=self.headers,
- cookies=self.cookies,
+ self.chat_endpoint,
+ headers=self.headers,
+ cookies=self.cookies,
params=params,
data=json.dumps(payload),
- stream=True,
+ stream=True,
timeout=self.timeout
)
if not response.ok:
@@ -312,11 +312,11 @@ def get_message(self, response: dict) -> str:
print("-" * 80)
print(f"{'Model':<50} {'Status':<10} {'Response'}")
print("-" * 80)
-
+
# Test all available models
working = 0
total = len(YouChat.AVAILABLE_MODELS)
-
+
for model in YouChat.AVAILABLE_MODELS:
try:
test_ai = YouChat(model=model, timeout=60)
@@ -325,7 +325,7 @@ def get_message(self, response: dict) -> str:
for chunk in response:
response_text += chunk
print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True)
-
+
if response_text and len(response_text.strip()) > 0:
status = "✓"
# Truncate response if too long
diff --git a/webscout/Provider/UNFINISHED/aihumanizer.py b/webscout/Provider/UNFINISHED/aihumanizer.py
index cfd4ec32..2b7a436c 100644
--- a/webscout/Provider/UNFINISHED/aihumanizer.py
+++ b/webscout/Provider/UNFINISHED/aihumanizer.py
@@ -1,6 +1,10 @@
-import requests
import uuid
+
+import requests
+from rich import print
+
from webscout.litagent import LitAgent
+
url = 'https://aihumanizer.work/api/v1/text/rewriter'
headers = {
@@ -33,5 +37,5 @@
}
response = requests.post(url, headers=headers, cookies=cookies, json=json_data)
-from rich import print
-print(response.json())
\ No newline at end of file
+
+print(response.json())
diff --git a/webscout/Provider/UNFINISHED/grammerchecker.py b/webscout/Provider/UNFINISHED/grammerchecker.py
index 90d6b9cf..21b9edba 100644
--- a/webscout/Provider/UNFINISHED/grammerchecker.py
+++ b/webscout/Provider/UNFINISHED/grammerchecker.py
@@ -1,5 +1,6 @@
import requests
+
def create_grammar_check_job(text: str):
url = 'https://api.aigrammarchecker.io/api/ai-check-grammar/create-job'
headers = {
@@ -32,4 +33,4 @@ def create_grammar_check_job(text: str):
if __name__ == "__main__":
from rich import print as cprint
- cprint(create_grammar_check_job("she gg to school"))
\ No newline at end of file
+ cprint(create_grammar_check_job("she gg to school"))
diff --git a/webscout/Provider/UNFINISHED/liner.py b/webscout/Provider/UNFINISHED/liner.py
index 75a7ec6c..cb4e83fc 100644
--- a/webscout/Provider/UNFINISHED/liner.py
+++ b/webscout/Provider/UNFINISHED/liner.py
@@ -1,15 +1,20 @@
-import requests
import json
import time
import uuid
from copy import deepcopy
-from typing import List, Dict, Optional, Union, Generator, Any
+from typing import Any, Dict, Generator, List, Optional, Union
+
+import requests
# Import base classes and utility structures
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
- ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta,
- ChatCompletionMessage, CompletionUsage
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ ChoiceDelta,
+ CompletionUsage,
)
# Attempt to import LitAgent, fallback if not available
@@ -46,7 +51,7 @@ def create(
if message.get("role") == "user":
user_content = message.get("content")
break
-
+
if not user_content:
raise ValueError("At least one user message is required")
@@ -54,7 +59,7 @@ def create(
payload = deepcopy(self._client.base_payload)
payload["query"] = user_content
payload["modelType"] = model
-
+
request_id = f"chatcmpl-{uuid.uuid4()}"
created_time = int(time.time())
@@ -84,21 +89,21 @@ def _create_stream(
continue
if not line.startswith("data:"):
continue
-
+
data_str = line[6:].strip()
if not data_str:
continue
-
+
try:
event = json.loads(data_str)
except json.JSONDecodeError:
continue
-
+
chunk_content = event.get("answer")
if chunk_content:
delta = ChoiceDelta(content=chunk_content, role="assistant")
choice = Choice(index=0, delta=delta, finish_reason=None)
-
+
chunk = ChatCompletionChunk(
id=request_id,
object="chat.completion.chunk",
@@ -107,7 +112,7 @@ def _create_stream(
choices=[choice]
)
yield chunk
-
+
# Send final chunk with finish_reason
final_delta = ChoiceDelta(content=None)
final_choice = Choice(index=0, delta=final_delta, finish_reason="stop")
@@ -145,16 +150,16 @@ def _create_non_stream(
continue
if not line.startswith("data:"):
continue
-
+
data_str = line[6:].strip()
if not data_str:
continue
-
+
try:
event = json.loads(data_str)
except json.JSONDecodeError:
continue
-
+
chunk = event.get("answer")
if chunk:
answer_parts.append(chunk)
@@ -163,7 +168,7 @@ def _create_non_stream(
raise IOError("No answer content received from Liner")
full_content = "".join(answer_parts)
-
+
message = ChatCompletionMessage(role="assistant", content=full_content)
choice = Choice(index=0, message=message, finish_reason="stop")
usage = CompletionUsage(
@@ -195,7 +200,7 @@ class Liner(OpenAICompatibleProvider):
Liner AI provider for OpenAI-compatible API.
Supports claude-4-5-sonnet and other models via Liner's search-enhanced AI.
"""
-
+
AVAILABLE_MODELS = [
"claude-4-5-sonnet",
"gpt-4",
@@ -211,7 +216,7 @@ def __init__(
):
"""
Initialize the Liner provider.
-
+
Args:
api_key: Not used, kept for compatibility
model: Model to use (default: claude-4-5-sonnet)
@@ -225,9 +230,9 @@ def __init__(
self.model = model
self.timeout = timeout
self.proxies = proxies
-
+
self.base_url = "https://getliner.com/lisa/v2/answer?lpv=250414"
-
+
self.headers = {
"accept": "text/event-stream",
"accept-language": "en-US,en;q=0.9,en-IN;q=0.8",
@@ -246,7 +251,7 @@ def __init__(
"sentry-trace": "30690146d0014887896af7e513702e97-92aa3c44e2ecea6b-0",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36 Edg/141.0.0.0",
}
-
+
self.cookies = {
"_k_state": "MmRiY2RjYTgtYjExMi00NTY1LWI1NWQtZjZkZjQ4NjFkZTE3LS0wNzZmNjVmNC05MTIzLTQ1MmItODQzMC0yYmRlYmEyNTA3NjQ=",
"__stripe_mid": "b3b3a7d0-d8a7-41b8-a75c-df41869803e0f06952",
@@ -269,7 +274,7 @@ def __init__(
"_dd_s": "aid=c19eef5b-af5a-4b9a-97b1-5d4ef1bfeff4&rum=0&expire=1760087072218",
"AMP_ac91207b66": "JTdCJTIyZGV2aWNlSWQlMjIlM0ElMjJiOTI5MzI0MC0wMTJmLTRlOTctYjUwYi03ZTdiNDIxM2RiZTAlMjIlMkMlMjJ1c2VySWQlMjIlM0ElMjI4NTI2MzE0JTIyJTJDJTIyc2Vzc2lvbklkJTIyJTNBMTc2MDA4NTQzOTg5NiUyQyUyMm9wdE91dCUyMiUzQWZhbHNlJTJDJTIybGFzdEV2ZW50VGltZSUyMiUzQTE3NjAwODYxNzIyMzclMkMlMjJsYXN0RXZlbnRJZCUyMiUzQTk3JTJDJTIycGFnZUNvdW50ZXIlMjIlM0E1JTdE",
}
-
+
self.base_payload = {
"spaceId": 17788115,
"threadId": 89019415,
@@ -288,7 +293,7 @@ def __init__(
"isDeepResearchMode": False,
"answerFormat": "auto",
}
-
+
self.session = requests.Session()
self.chat = Chat(self)
@@ -307,7 +312,7 @@ def list(inner_self):
if __name__ == "__main__":
# Example usage
client = Liner(model="claude-4-5-sonnet")
-
+
# Non-streaming example
response = client.chat.completions.create(
model="claude-4-5-sonnet",
@@ -318,7 +323,7 @@ def list(inner_self):
)
print("Non-streaming response:")
print(response.choices[0].message.content)
-
+
# Streaming example
print("\nStreaming response:")
stream = client.chat.completions.create(
diff --git a/webscout/Provider/UNFINISHED/liner_api_request.py b/webscout/Provider/UNFINISHED/liner_api_request.py
index 88fdfd01..36531969 100644
--- a/webscout/Provider/UNFINISHED/liner_api_request.py
+++ b/webscout/Provider/UNFINISHED/liner_api_request.py
@@ -1,25 +1,23 @@
-import requests
import json
-from typing import Dict, Optional, Generator, Union, Any
-from uuid import uuid4
-import time
-import base64
import random
+from typing import Dict, Generator, Optional, Union
+from uuid import uuid4
+
+import requests
+from webscout import LitAgent, exceptions
from webscout.AIbase import AISearch
-from webscout import exceptions
-from webscout import LitAgent
class Response:
"""A wrapper class for Liner API responses.
-
+
This class automatically converts response objects to their text representation
when printed or converted to string.
-
+
Attributes:
text (str): The text content of the response
-
+
Example:
>>> response = Response("Hello, world!")
>>> print(response)
@@ -29,20 +27,20 @@ class Response:
"""
def __init__(self, text: str):
self.text = text
-
+
def __str__(self):
return self.text
-
+
def __repr__(self):
return self.text
class Liner(AISearch):
"""A class to interact with the Liner AI search API.
-
+
Liner provides a powerful search interface that returns AI-generated responses
based on web content. It supports both streaming and non-streaming responses.
-
+
Basic Usage:
>>> from webscout import Liner
>>> ai = Liner(cookies_path="cookies.json")
@@ -50,18 +48,18 @@ class Liner(AISearch):
>>> response = ai.search("What is Python?")
>>> print(response)
Python is a high-level programming language...
-
+
>>> # Streaming example
>>> for chunk in ai.search("Tell me about AI", stream=True):
... print(chunk, end="", flush=True)
Artificial Intelligence is...
-
+
>>> # Raw response format
>>> for chunk in ai.search("Hello", stream=True, raw=True):
... print(chunk)
{'text': 'Hello'}
{'text': ' there!'}
-
+
Args:
cookies_path (str): Path to the cookies JSON file
timeout (int, optional): Request timeout in seconds. Defaults to 30.
@@ -79,7 +77,7 @@ def __init__(
reasoning_mode: bool = False,
):
"""Initialize the Liner API client.
-
+
Args:
cookies_path (str): Path to the cookies JSON file
timeout (int, optional): Request timeout in seconds. Defaults to 30.
@@ -95,13 +93,13 @@ def __init__(
self.cookies_path = cookies_path
self.deep_search = deep_search
self.reasoning_mode = reasoning_mode
-
+
# Generate random IDs
self.space_id = random.randint(10000000, 99999999)
self.thread_id = random.randint(10000000, 99999999)
self.user_message_id = random.randint(100000000, 999999999)
self.user_id = random.randint(1000000, 9999999)
-
+
self.headers = {
"accept": "text/event-stream",
"accept-encoding": "gzip, deflate, br, zstd",
@@ -119,7 +117,7 @@ def __init__(
"sec-gpc": "1",
"user-agent": LitAgent().random()
}
-
+
# Load cookies from JSON file
self.cookies = self._load_cookies()
if not self.cookies:
@@ -132,7 +130,7 @@ def __init__(
def _load_cookies(self) -> Optional[Dict[str, str]]:
"""Load cookies from a JSON file.
-
+
Returns:
Optional[Dict[str, str]]: Dictionary of cookies if successful, None otherwise
"""
@@ -157,7 +155,7 @@ def search(
raw: bool = False,
) -> Union[Response, Generator[Union[Dict[str, str], Response], None, None]]:
"""Search using the Liner API and get AI-generated responses.
-
+
Args:
prompt (str): The search query or prompt to send to the API.
stream (bool, optional): If True, yields response chunks as they arrive.
@@ -165,12 +163,12 @@ def search(
raw (bool, optional): If True, returns raw response dictionaries with 'text' key.
If False, returns Response objects that convert to text automatically.
Defaults to False.
-
+
Returns:
- Union[Response, Generator[Union[Dict[str, str], Response], None, None]]:
+ Union[Response, Generator[Union[Dict[str, str], Response], None, None]]:
- If stream=False: Returns complete response
- If stream=True: Yields response chunks as they arrive
-
+
Raises:
APIConnectionError: If the API request fails
"""
@@ -192,7 +190,7 @@ def search(
"experimentVariants": [],
"isDeepResearchMode": self.deep_search
}
-
+
def for_stream():
try:
with self.session.post(
@@ -205,18 +203,18 @@ def for_stream():
raise exceptions.APIConnectionError(
f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}"
)
-
+
current_reasoning = ""
current_answer = ""
-
+
for line in response.iter_lines(decode_unicode=True):
if line == "event:finish_answer":
break
-
+
if line.startswith('data:'):
try:
data = json.loads(line[5:]) # Remove 'data:' prefix
-
+
# Handle reasoning updates if enabled
if self.reasoning_mode and 'reasoning' in data:
current_reasoning += data['reasoning']
@@ -224,7 +222,7 @@ def for_stream():
yield {"text": data['reasoning']}
else:
yield Response(data['reasoning'])
-
+
# Handle answer updates
if 'answer' in data:
current_answer += data['answer']
@@ -232,13 +230,13 @@ def for_stream():
yield {"text": data['answer']}
else:
yield Response(data['answer'])
-
+
except json.JSONDecodeError:
continue
-
+
except requests.exceptions.RequestException as e:
raise exceptions.APIConnectionError(f"Request failed: {e}")
-
+
def for_non_stream():
full_response = ""
for chunk in for_stream():
@@ -246,7 +244,7 @@ def for_non_stream():
yield chunk
else:
full_response += str(chunk)
-
+
if not raw:
self.last_response = Response(full_response)
return self.last_response
@@ -256,8 +254,8 @@ def for_non_stream():
if __name__ == "__main__":
from rich import print
-
+
ai = Liner(cookies_path="cookies.json")
response = ai.search(input(">>> "), stream=True, raw=False)
for chunk in response:
- print(chunk, end="", flush=True)
\ No newline at end of file
+ print(chunk, end="", flush=True)
diff --git a/webscout/Provider/UNFINISHED/samurai.py b/webscout/Provider/UNFINISHED/samurai.py
index 8a9d7cca..a1339c39 100644
--- a/webscout/Provider/UNFINISHED/samurai.py
+++ b/webscout/Provider/UNFINISHED/samurai.py
@@ -1,12 +1,12 @@
-from typing import *
-from webscout.AIutel import Conversation
-from webscout.AIutel import Optimizers
-from webscout.AIutel import AwesomePrompts, sanitize_stream
-from webscout.AIbase import Provider
-from webscout import exceptions
-from curl_cffi.requests import Session
-from curl_cffi import CurlError
import json
+from typing import Any, Dict, Generator, Optional, Union
+
+from curl_cffi.requests import Session
+
+from webscout import exceptions
+from webscout.AIbase import Provider
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
+
class samurai(Provider):
"""
@@ -221,4 +221,4 @@ def get_message(self, response: dict) -> str:
display_text = "Empty or invalid response"
print(f"\r{model:<50} {status:<10} {display_text}")
except Exception as e:
- print(f"\r{model:<50} {'✗':<10} {str(e)}")
\ No newline at end of file
+ print(f"\r{model:<50} {'✗':<10} {str(e)}")
diff --git a/webscout/Provider/Venice.py b/webscout/Provider/Venice.py
index 25bfae8a..a1820978 100644
--- a/webscout/Provider/Venice.py
+++ b/webscout/Provider/Venice.py
@@ -1,29 +1,33 @@
-from curl_cffi import CurlError
-from curl_cffi.requests import Session # Import Session
-import json
-from typing import Generator, Dict, Any, List, Optional, Union
-from uuid import uuid4
import random
+from typing import Any, Dict, Generator, Optional, Union
+from uuid import uuid4
+
+from curl_cffi import CurlError
+from curl_cffi.requests import Session # Import Session
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import ( # Import sanitize_stream
+ AwesomePrompts,
+ Conversation,
+ Optimizers,
+ sanitize_stream,
+)
from webscout.litagent import LitAgent
+
class Venice(Provider):
"""
A class to interact with the Venice AI API.
"""
-
+
required_auth = False
AVAILABLE_MODELS = [
"mistral-31-24b",
"dolphin-3.0-mistral-24b",
"dolphin-3.0-mistral-24b-1dot1"
]
-
+
def __init__(
self,
is_conversation: bool = True,
@@ -31,24 +35,24 @@ def __init__(
timeout: int = 30,
temperature: float = 0.8, # Keep temperature, user might want to adjust
top_p: float = 0.9, # Keep top_p
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
- model: str = "mistral-31-24b",
+ act: Optional[str] = None,
+ model: str = "mistral-31-24b",
# System prompt is empty in the example, but keep it configurable
- system_prompt: str = ""
+ system_prompt: str = ""
):
"""Initialize Venice AI client"""
if model not in self.AVAILABLE_MODELS:
raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}")
-
+
# Update API endpoint
- self.api_endpoint = "https://outerface.venice.ai/api/inference/chat"
+ self.api_endpoint = "https://outerface.venice.ai/api/inference/chat"
# Initialize curl_cffi Session
- self.session = Session()
+ self.session = Session()
self.is_conversation = is_conversation
self.max_tokens_to_sample = max_tokens
self.temperature = temperature
@@ -57,7 +61,7 @@ def __init__(
self.model = model
self.system_prompt = system_prompt
self.last_response = {}
-
+
# Update Headers based on successful request
self.headers = {
"User-Agent": LitAgent().random(), # Keep using LitAgent
@@ -67,15 +71,15 @@ def __init__(
"origin": "https://venice.ai",
"referer": "https://venice.ai/", # Update referer
# Update sec-ch-ua to match example
- "sec-ch-ua": '"Microsoft Edge";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
+ "sec-ch-ua": '"Microsoft Edge";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"Windows"',
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
# Update sec-fetch-site to match example
- "sec-fetch-site": "same-site",
+ "sec-fetch-site": "same-site",
# Add missing headers from example
- "priority": "u=1, i",
+ "priority": "u=1, i",
"sec-gpc": "1",
"x-venice-version": "interface@20250424.065523+50bac27" # Add version header
}
@@ -83,7 +87,7 @@ def __init__(
# Update curl_cffi session headers and proxies
self.session.headers.update(self.headers)
self.session.proxies.update(proxies)
-
+
self.__available_optimizers = (
method
for method in dir(Optimizers)
@@ -115,9 +119,10 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Union[Dict[str, Any], Generator]:
+ **kwargs: Any,
+ ) -> Response:
conversation_prompt = self.conversation.gen_complete_prompt(prompt)
if optimizer:
if optimizer in self.__available_optimizers:
@@ -145,12 +150,12 @@ def ask(
def for_stream():
try:
response = self.session.post(
- self.api_endpoint,
- json=payload,
- stream=True,
+ self.api_endpoint,
+ json=payload,
+ stream=True,
timeout=self.timeout,
impersonate="edge101"
- )
+ )
if response.status_code != 200:
raise exceptions.FailedToGenerateResponseError(
f"Request failed with status code {response.status_code} - {response.text}"
@@ -175,30 +180,30 @@ def for_stream():
streaming_text += content_chunk
yield dict(text=content_chunk)
self.conversation.update_chat_history(prompt, streaming_text)
- self.last_response = {"text": streaming_text}
- except CurlError as e:
+ self.last_response = {"text": streaming_text}
+ except CurlError as e:
raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {e}")
- except Exception as e:
+ except Exception as e:
raise exceptions.FailedToGenerateResponseError(f"An unexpected error occurred ({type(e).__name__}): {e}")
def for_non_stream():
full_text = ""
- for chunk_data in for_stream():
+ for chunk_data in for_stream():
if isinstance(chunk_data, dict) and "text" in chunk_data:
full_text += chunk_data["text"]
- elif isinstance(chunk_data, str):
+ elif isinstance(chunk_data, str):
full_text += chunk_data
- self.last_response = {"text": full_text}
- return self.last_response
+ self.last_response = {"text": full_text}
+ return self.last_response
return for_stream() if stream else for_non_stream()
def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
raw: bool = False, # Added raw parameter
- ) -> Union[str, Generator]:
+ ) -> Union[str, Generator[str, None, None]]:
def for_stream():
for response in self.ask(prompt, True, raw=raw, optimizer=optimizer, conversationally=conversationally):
if raw:
@@ -222,20 +227,23 @@ def get_message(self, response: dict) -> str:
print("-" * 80)
print(f"{'Model':<50} {'Status':<10} {'Response'}")
print("-" * 80)
-
+
# Test all available models
working = 0
total = len(Venice.AVAILABLE_MODELS)
-
+
for model in Venice.AVAILABLE_MODELS:
try:
test_ai = Venice(model=model, timeout=60)
response = test_ai.chat("Say 'Hello' in one word", stream=True)
response_text = ""
- for chunk in response:
- response_text += chunk
- print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True)
-
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)):
+ for chunk in response:
+ response_text += chunk
+ print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True)
+ else:
+ response_text = str(response)
+
if response_text and len(response_text.strip()) > 0:
status = "✓"
# Truncate response if too long
@@ -245,4 +253,4 @@ def get_message(self, response: dict) -> str:
display_text = "Empty or invalid response"
print(f"\r{model:<50} {status:<10} {display_text}")
except Exception as e:
- print(f"\r{model:<50} {'✗':<10} {str(e)}")
\ No newline at end of file
+ print(f"\r{model:<50} {'✗':<10} {str(e)}")
diff --git a/webscout/Provider/VercelAI.py b/webscout/Provider/VercelAI.py
index 7786d58e..abc237f4 100644
--- a/webscout/Provider/VercelAI.py
+++ b/webscout/Provider/VercelAI.py
@@ -1,15 +1,19 @@
+import json
import re
import time
-from curl_cffi import requests
-import json
-from typing import Union, Any, Dict, Generator, Optional
import uuid
+from typing import Any, Dict, Generator, Optional, Union
+
+from curl_cffi import requests
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import ( # Import sanitize_stream
+ AwesomePrompts,
+ Conversation,
+ Optimizers,
+ sanitize_stream,
+)
from webscout.litagent import LitAgent
@@ -29,12 +33,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 600,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
model: str = "chat-model",
system_prompt: str = "You are a helpful AI assistant."
):
@@ -128,9 +132,10 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Union[Dict[str, Any], Generator[Any, None, None]]:
+ **kwargs: Any,
+ ) -> Response:
"""Chat with AI"""
conversation_prompt = self.conversation.gen_complete_prompt(prompt)
if optimizer:
@@ -194,10 +199,10 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
raw: bool = False, # Added raw parameter
- ) -> str:
+ ) -> Union[str, Generator[str, None, None]]:
def for_stream():
for response in self.ask(
prompt, True, raw=raw, optimizer=optimizer, conversationally=conversationally
@@ -222,7 +227,7 @@ def for_non_stream():
def get_message(self, response: dict) -> str:
"""Retrieves message only from response"""
- assert isinstance(response, dict), "Response should be of dict data-type only"
+ assert isinstance(response, dict), "Response should be of dict data-type only"
# Formatting is handled by the extractor now
text = response.get("text", "")
return text.replace('\\n', '\n').replace('\\n\\n', '\n\n') # Keep newline replacement if needed
@@ -231,20 +236,23 @@ def get_message(self, response: dict) -> str:
print("-" * 80)
print(f"{'Model':<50} {'Status':<10} {'Response'}")
print("-" * 80)
-
+
# Test all available models
working = 0
total = len(VercelAI.AVAILABLE_MODELS)
-
+
for model in VercelAI.AVAILABLE_MODELS:
try:
test_ai = VercelAI(model=model, timeout=60)
response = test_ai.chat("Say 'Hello' in one word", stream=True)
response_text = ""
- for chunk in response:
- response_text += chunk
- print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True)
-
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)):
+ for chunk in response:
+ response_text += chunk
+ print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True)
+ else:
+ response_text = str(response)
+
if response_text and len(response_text.strip()) > 0:
status = "✓"
# Truncate response if too long
@@ -254,4 +262,4 @@ def get_message(self, response: dict) -> str:
display_text = "Empty or invalid response"
print(f"\r{model:<50} {status:<10} {display_text}")
except Exception as e:
- print(f"\r{model:<50} {'✗':<10} {str(e)}")
\ No newline at end of file
+ print(f"\r{model:<50} {'✗':<10} {str(e)}")
diff --git a/webscout/Provider/WiseCat.py b/webscout/Provider/WiseCat.py
index 6ee7e7e5..0130ff2d 100644
--- a/webscout/Provider/WiseCat.py
+++ b/webscout/Provider/WiseCat.py
@@ -1,14 +1,16 @@
-import re
-import json
-from typing import Union, Any, Dict, Generator, Optional
-from curl_cffi import CurlError
+from typing import Any, Dict, Generator, Optional, Union
+
+from curl_cffi import CurlError
from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import ( # Import sanitize_stream
+ AwesomePrompts,
+ Conversation,
+ Optimizers,
+ sanitize_stream,
+)
from webscout.litagent import LitAgent
@@ -28,12 +30,12 @@ def __init__(self,
is_conversation: bool = True,
max_tokens: int = 600,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
model: str = "chat-model-small",
system_prompt: str = "You are a helpful AI assistant."
):
@@ -48,14 +50,14 @@ def __init__(self,
self.max_tokens_to_sample = max_tokens
self.api_endpoint = "https://wise-cat-groq.vercel.app/api/chat"
# stream_chunk_size is not directly applicable to curl_cffi iter_lines
- # self.stream_chunk_size = 64
+ # self.stream_chunk_size = 64
self.timeout = timeout
self.last_response = {}
self.model = model
self.system_prompt = system_prompt
self.litagent = LitAgent()
# Generate headers using LitAgent, but apply them to the curl_cffi session
- self.headers = self.litagent.generate_fingerprint()
+ self.headers = self.litagent.generate_fingerprint()
# Update curl_cffi session headers and proxies
self.session.headers.update(self.headers)
self.session.proxies = proxies
@@ -83,9 +85,10 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Union[Dict[str, Any], Generator[Any, None, None]]:
+ **kwargs: Any,
+ ) -> Response:
"""Chat with AI"""
conversation_prompt = self.conversation.gen_complete_prompt(prompt)
if optimizer:
@@ -114,10 +117,10 @@ def ask(
def for_stream():
try:
response = self.session.post(
- self.api_endpoint,
- headers=self.headers,
- json=payload,
- stream=True,
+ self.api_endpoint,
+ headers=self.headers,
+ json=payload,
+ stream=True,
timeout=self.timeout,
impersonate="chrome120"
)
@@ -144,7 +147,7 @@ def for_stream():
# Handle unicode escaping and quote unescaping
extracted_content = content_chunk.encode().decode('unicode_escape')
extracted_content = extracted_content.replace('\\\\', '\\').replace('\\"', '"')
-
+
if raw:
yield extracted_content
else:
@@ -168,10 +171,10 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
raw: bool = False, # Added raw parameter
- ) -> str:
+ ) -> Union[str, Generator[str, None, None]]:
def for_stream():
for response in self.ask(
prompt, True, raw=raw, optimizer=optimizer, conversationally=conversationally
@@ -206,20 +209,23 @@ def get_message(self, response: dict) -> str:
print("-" * 80)
print(f"{'Model':<50} {'Status':<10} {'Response'}")
print("-" * 80)
-
+
# Test all available models
working = 0
total = len(WiseCat.AVAILABLE_MODELS)
-
+
for model in WiseCat.AVAILABLE_MODELS:
try:
test_ai = WiseCat(model=model, timeout=60)
response = test_ai.chat("Say 'Hello' in one word", stream=True)
response_text = ""
- for chunk in response:
- response_text += chunk
- print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True)
-
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)):
+ for chunk in response:
+ response_text += chunk
+ print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True)
+ else:
+ response_text = str(response)
+
if response_text and len(response_text.strip()) > 0:
status = "✓"
# Truncate response if too long
@@ -229,4 +235,4 @@ def get_message(self, response: dict) -> str:
display_text = "Empty or invalid response"
print(f"\r{model:<50} {status:<10} {display_text}")
except Exception as e:
- print(f"\r{model:<50} {'✗':<10} {str(e)}")
\ No newline at end of file
+ print(f"\r{model:<50} {'✗':<10} {str(e)}")
diff --git a/webscout/Provider/WrDoChat.py b/webscout/Provider/WrDoChat.py
index 0cb95930..68401982 100644
--- a/webscout/Provider/WrDoChat.py
+++ b/webscout/Provider/WrDoChat.py
@@ -1,18 +1,18 @@
import json
import re
-from typing import Optional, Union, Any, Dict, Generator
from datetime import datetime
+from typing import Any, Dict, Generator, Optional, Union
from uuid import uuid4
+
from curl_cffi import CurlError
from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
from webscout.litagent import LitAgent
+
class WrDoChat(Provider):
"""
A class to interact with the oi.wr.do chat API.
@@ -58,12 +58,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 2000,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
model: str = "gemini-2.5-flash-preview-04-17",
system_prompt: str = "You are a helpful AI assistant.",
):
@@ -149,7 +149,7 @@ def _load_cookies(self) -> Optional[Dict[str, str]]:
def _wrdo_extractor(self, line: Union[str, Dict[str, Any]]) -> Optional[str]:
"""Extracts content from the oi.wr.do stream format.
-
+
Format:
f:{"messageId":"..."}
0:"content chunk"
@@ -163,7 +163,7 @@ def _wrdo_extractor(self, line: Union[str, Dict[str, Any]]) -> Optional[str]:
# Decode potential unicode escapes like \u00e9
content = match.group(1).encode().decode('unicode_escape')
return content.replace('\\\\', '\\').replace('\\"', '"') # Handle escaped backslashes and quotes
-
+
# Store message ID from 'f:' response
elif line.startswith('f:'):
try:
@@ -188,9 +188,10 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Union[Dict[str, Any], Generator[Dict[str, Any], None, None]]:
+ **kwargs: Any,
+ ) -> Response:
"""
Send a message to the oi.wr.do API.
@@ -217,7 +218,8 @@ def ask(
chat_id = str(uuid4())
message_id = str(uuid4())
- current_time = datetime.utcnow().isoformat() + "Z"
+ from datetime import timezone
+ current_time = datetime.now(timezone.utc).isoformat() + "Z"
payload = {
"id": chat_id,
@@ -304,7 +306,7 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
raw: bool = False, # Added raw parameter
) -> Union[str, Generator[str, None, None]]:
@@ -342,26 +344,31 @@ def for_non_stream():
return self.get_message(result)
return for_stream() if stream else for_non_stream()
- def get_message(self, response: dict) -> str:
+ def get_message(self, response: Response) -> str:
"""
Extract message from response.
Args:
- response (dict): The response dictionary.
+ response (Response): The response dictionary.
Returns:
str: The extracted message.
"""
- assert isinstance(response, dict), "Response should be of dict data-type only"
+ if not isinstance(response, dict):
+ return str(response)
return response.get("text", "")
if __name__ == "__main__":
- from rich import print
import json
-
+
+ from rich import print
+
# Example usage
ai = WrDoChat(cookies_path="cookies.json")
response = ai.chat("write me a poem about AI", stream=True)
- for chunk in response:
- print(chunk, end="", flush=True)
\ No newline at end of file
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)):
+ for chunk in response:
+ print(chunk, end="", flush=True)
+ else:
+ print(response)
diff --git a/webscout/Provider/__init__.py b/webscout/Provider/__init__.py
index 808494f9..d7f812e7 100644
--- a/webscout/Provider/__init__.py
+++ b/webscout/Provider/__init__.py
@@ -5,6 +5,7 @@
from webscout.Provider.Algion import Algion
from webscout.Provider.Andi import AndiSearch
from webscout.Provider.Apriel import Apriel
+from webscout.Provider.Ayle import Ayle
from webscout.Provider.cerebras import Cerebras
from webscout.Provider.ChatSandbox import ChatSandbox
from webscout.Provider.ClaudeOnline import ClaudeOnline
@@ -15,13 +16,12 @@
from webscout.Provider.elmo import Elmo
from webscout.Provider.EssentialAI import EssentialAI
from webscout.Provider.ExaAI import ExaAI
-from webscout.Provider.Ayle import Ayle
from webscout.Provider.Gemini import GEMINI
from webscout.Provider.geminiapi import GEMINIAPI
from webscout.Provider.GithubChat import GithubChat
-from webscout.Provider.HadadXYZ import HadadXYZ
from webscout.Provider.Gradient import Gradient
from webscout.Provider.Groq import GROQ
+from webscout.Provider.HadadXYZ import HadadXYZ
from webscout.Provider.HeckAI import HeckAI
from webscout.Provider.HuggingFace import HuggingFace
from webscout.Provider.IBM import IBM
@@ -37,7 +37,6 @@
from webscout.Provider.Netwrck import Netwrck
from webscout.Provider.Nvidia import Nvidia
from webscout.Provider.oivscode import oivscode
-from .Openai import OPENAI
from webscout.Provider.PI import PiAI
from webscout.Provider.QwenLM import QwenLM
from webscout.Provider.Sambanova import Sambanova
@@ -55,7 +54,8 @@
from webscout.Provider.WiseCat import WiseCat
from webscout.Provider.WrDoChat import WrDoChat
from webscout.Provider.x0gpt import X0GPT
-from webscout.Provider.yep import YEPCHAT
+
+from .Openai import OPENAI
# List of all exported names
__all__ = [
@@ -114,5 +114,4 @@
"WiseCat",
"WrDoChat",
"X0GPT",
- "YEPCHAT",
]
diff --git a/webscout/Provider/ai4chat.py b/webscout/Provider/ai4chat.py
index 105b5559..17ecee01 100644
--- a/webscout/Provider/ai4chat.py
+++ b/webscout/Provider/ai4chat.py
@@ -1,11 +1,11 @@
-from curl_cffi.requests import Session, RequestsError
import urllib.parse
-from typing import Union, Any, Dict
+from typing import Any, Generator, Optional, Union
+
+from curl_cffi.requests import RequestsError, Session
+
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts
-from webscout.AIbase import Provider
class AI4Chat(Provider):
"""
@@ -17,12 +17,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 600,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
system_prompt: str = "You are a helpful and informative AI assistant.",
country: str = "Asia",
user_id: str = "usersmjb2oaz7y"
@@ -65,18 +65,19 @@ def __init__(
is_conversation, self.max_tokens_to_sample, filepath, update_file
)
self.conversation.history_offset = history_offset
- self.system_prompt = system_prompt
+ self.system_prompt = system_prompt
def ask(
self,
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- country: str = None,
- user_id: str = None,
- ):
+ country: Optional[str] = None,
+ user_id: Optional[str] = None,
+ **kwargs: Any,
+ ) -> Response:
"""
Sends a prompt to the AI4Chat API and returns the response.
If stream=True, yields small chunks of the response (simulated streaming).
@@ -127,11 +128,12 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- country: str = None,
- user_id: str = None,
- ):
+ country: Optional[str] = None,
+ user_id: Optional[str] = None,
+ **kwargs: Any,
+ ) -> Union[str, Generator[str, None, None]]:
"""
Generates a response from the AI4Chat API.
If stream=True, yields each chunk as a string.
@@ -168,7 +170,10 @@ def get_message(self, response: Union[dict, str]) -> str:
if __name__ == "__main__":
from rich import print
- ai = AI4Chat()
+ ai = AI4Chat()
response = ai.chat("Tell me about humans in points", stream=True)
- for c in response:
- print(c, end="")
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)):
+ for c in response:
+ print(c, end="")
+ else:
+ print(response)
diff --git a/webscout/Provider/akashgpt.py b/webscout/Provider/akashgpt.py
index a6576c1a..1f4e8f57 100644
--- a/webscout/Provider/akashgpt.py
+++ b/webscout/Provider/akashgpt.py
@@ -1,17 +1,21 @@
-from typing import Optional, Union, Any, Dict, Generator
-from uuid import uuid4
-import cloudscraper
import re
-import json
import time
+from typing import Any, Dict, Generator, Optional, Union
+from uuid import uuid4
+
+import cloudscraper
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import ( # Import sanitize_stream
+ AwesomePrompts,
+ Conversation,
+ Optimizers,
+ sanitize_stream,
+)
from webscout.litagent import LitAgent
+
class AkashGPT(Provider):
"""
A class to interact with the Akash Network Chat API.
@@ -40,12 +44,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 600,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
system_prompt: str = "You are a helpful assistant.",
model: str = "meta-llama-3-3-70b-instruct",
temperature: float = 0.6,
@@ -73,7 +77,7 @@ def __init__(
# Validate model choice
if model not in self.AVAILABLE_MODELS:
raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}")
-
+
self.session = cloudscraper.create_scraper()
self.is_conversation = is_conversation
self.max_tokens_to_sample = max_tokens
@@ -113,7 +117,7 @@ def __init__(
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"sec-gpc": "1",
- "user-agent": self.agent.random()
+ "user-agent": self.agent.random()
}
# Set cookies with the session token
@@ -154,9 +158,10 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Dict[str, Any]:
+ **kwargs: Any,
+ ) -> Response:
"""
Sends a prompt to the Akash Network API and returns the response.
@@ -250,9 +255,10 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> str:
+ **kwargs: Any,
+ ) -> Union[str, Generator[str, None, None]]:
"""
Generates a response from the AkashGPT API.
@@ -319,8 +325,12 @@ def get_message(self, response: dict) -> str:
try:
test_ai = AkashGPT(model=model, timeout=60, api_key="5ef9b0782df982fab720810f6ee72a9af01ebadbd9eb05adae0ecc8711ec79c5; _ga_LFRGN2J2RV=GS2.1.s1763554272$o4$g1$t1763554284$j48$l0$h0") # Example key
response = test_ai.chat("Say 'Hello' in one word")
- response_text = response
-
+
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)):
+ response_text = "".join(list(response))
+ else:
+ response_text = str(response)
+
if response_text and len(response_text.strip()) > 0:
status = "✓"
# Truncate response if too long
@@ -330,4 +340,4 @@ def get_message(self, response: dict) -> str:
display_text = "Empty or invalid response"
print(f"{model:<50} {status:<10} {display_text}")
except Exception as e:
- print(f"{model:<50} {'✗':<10} {str(e)}")
\ No newline at end of file
+ print(f"{model:<50} {'✗':<10} {str(e)}")
diff --git a/webscout/Provider/cerebras.py b/webscout/Provider/cerebras.py
index 15a8368e..b27dbb00 100644
--- a/webscout/Provider/cerebras.py
+++ b/webscout/Provider/cerebras.py
@@ -1,6 +1,5 @@
import re
-
# Import trio before curl_cffi to prevent eventlet socket monkey-patching conflicts
# See: https://github.com/python-trio/trio/issues/3015
try:
@@ -14,7 +13,7 @@
from curl_cffi.requests import Session
from webscout import exceptions
-from webscout.AIbase import Provider
+from webscout.AIbase import Provider, Response
from webscout.AIutel import ( # Import sanitize_stream
AwesomePrompts,
Conversation,
@@ -42,18 +41,18 @@ class Cerebras(Provider):
]
@classmethod
- def get_models(cls, api_key: str = None):
+ def get_models(cls, api_key: Optional[str] = None):
"""Fetch available models from Cerebras API.
-
+
Args:
api_key (str, optional): Cerebras API key. If not provided, returns default models.
-
+
Returns:
list: List of available model IDs
"""
if not api_key:
raise Exception("API key required to fetch models")
-
+
try:
# Use a temporary curl_cffi session for this class method
temp_session = Session()
@@ -61,37 +60,37 @@ def get_models(cls, api_key: str = None):
"Content-Type": "application/json",
"Authorization": f"Bearer {api_key}",
}
-
+
response = temp_session.get(
"https://api.cerebras.ai/v1/models",
headers=headers,
impersonate="chrome120"
)
-
+
if response.status_code != 200:
raise Exception(f"Failed to fetch models: HTTP {response.status_code}")
-
+
data = response.json()
if "data" in data and isinstance(data["data"], list):
return [model['id'] for model in data['data']]
raise Exception("Invalid response format from API")
-
+
except (curl_cffi.CurlError, Exception) as e:
raise Exception(f"Failed to fetch models: {str(e)}")
def __init__(
self,
- cookie_path: str = None,
+ cookie_path: Optional[str] = None,
is_conversation: bool = True,
max_tokens: int = 40000,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
- api_key: str = None,
+ act: Optional[str] = None,
+ api_key: Optional[str] = None,
model: str = "qwen-3-coder-480b",
system_prompt: str = "You are a helpful assistant.",
temperature: float = 0.7,
@@ -296,9 +295,10 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False, # Add raw parameter for consistency
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Union[Dict, Generator]:
+ **kwargs: Any,
+ ) -> Response:
"""Send a prompt to the model and get a response."""
conversation_prompt = self.conversation.gen_complete_prompt(prompt)
if optimizer:
@@ -341,10 +341,11 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
raw: bool = False,
- ) -> Union[str, Generator]:
+ **kwargs: Any,
+ ) -> Union[str, Generator[str, None, None]]:
"""Chat with the model."""
# Ask returns a generator for stream=True, dict/str for stream=False
response_gen_or_dict = self.ask(prompt, stream, raw=raw, optimizer=optimizer, conversationally=conversationally)
@@ -364,10 +365,11 @@ def stream_wrapper():
return response_gen_or_dict
return self.get_message(response_gen_or_dict)
- def get_message(self, response: str) -> str:
+ def get_message(self, response: Response) -> str:
"""Retrieves message from response."""
# Updated to handle dict input from ask()
- assert isinstance(response, dict), "Response should be of dict data-type only for get_message"
+ if not isinstance(response, dict):
+ return str(response)
return response.get("text", "")
if __name__ == "__main__":
diff --git a/webscout/Provider/cleeai.py b/webscout/Provider/cleeai.py
index 4bc41459..de0e0ebe 100644
--- a/webscout/Provider/cleeai.py
+++ b/webscout/Provider/cleeai.py
@@ -1,14 +1,14 @@
-import requests
import json
+from typing import Any, Generator, Optional, Union
from uuid import uuid4
-from typing import Union, Dict, Any, Generator
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream
-from webscout.AIbase import Provider
-from webscout import exceptions
+import requests
+
import webscout
+from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
+
class Cleeai(Provider):
"""
@@ -20,12 +20,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 600,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
) -> None:
"""Instantiates Cleeai
@@ -89,9 +89,10 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> dict:
+ **kwargs: Any,
+ ) -> Response:
"""Chat with AI
Args:
@@ -178,7 +179,7 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
raw: bool = False,
) -> Union[str, Generator[str, None, None]]:
@@ -233,5 +234,8 @@ def get_message(self, response: dict) -> str:
from rich import print
ai = Cleeai(timeout=5000)
response = ai.chat("tell me about Abhay koul, HelpingAI", stream=True)
- for chunk in response:
- print(chunk, end="", flush=True)
\ No newline at end of file
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)):
+ for chunk in response:
+ print(chunk, end="", flush=True)
+ else:
+ print(response)
diff --git a/webscout/Provider/elmo.py b/webscout/Provider/elmo.py
index 8f00d16f..d5ca000d 100644
--- a/webscout/Provider/elmo.py
+++ b/webscout/Provider/elmo.py
@@ -1,14 +1,17 @@
-from curl_cffi.requests import Session
+import re # Import re for the extractor
+from typing import Any, Dict, Generator, Optional, Union
+
from curl_cffi import CurlError
-import json
-from typing import Optional, Union, Any, Dict, Generator
+from curl_cffi.requests import Session
+
from webscout import exceptions
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation, sanitize_stream # Import sanitize_stream
-from webscout.AIutel import AwesomePrompts
-from webscout.AIbase import Provider
-from webscout.litagent import LitAgent
-import re # Import re for the extractor
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import ( # Import sanitize_stream
+ AwesomePrompts,
+ Conversation,
+ Optimizers,
+ sanitize_stream,
+)
class Elmo(Provider):
@@ -21,12 +24,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 600, # Note: max_tokens is not used by this API
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
system_prompt: str = "You are a helpful AI assistant. Provide clear, concise, and well-structured information. Organize your responses into paragraphs for better readability.",
) -> None:
"""Instantiates Elmo
@@ -99,11 +102,12 @@ def _elmo_extractor(chunk: Union[str, Dict[str, Any]]) -> Optional[str]:
def ask(
self,
prompt: str,
- stream: bool = False, # API supports streaming
+ stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Union[Dict[str, Any], Generator[Any, None, None]]: # Corrected return type hint
+ **kwargs: Any,
+ ) -> Response:
"""Chat with AI
Args:
@@ -230,9 +234,10 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Union[str, Generator[str, None, None]]: # Corrected return type hint
+ **kwargs: Any,
+ ) -> Union[str, Generator[str, None, None]]:
"""Generate response `str`
Args:
prompt (str): Prompt to be send.
@@ -283,5 +288,8 @@ def get_message(self, response: dict) -> str:
from rich import print
ai = Elmo()
response = ai.chat("write a poem about AI", stream=True)
- for chunk in response:
- print(chunk, end="", flush=True)
\ No newline at end of file
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)):
+ for chunk in response:
+ print(chunk, end="", flush=True)
+ else:
+ print(response)
diff --git a/webscout/Provider/geminiapi.py b/webscout/Provider/geminiapi.py
index 80aa18e9..9e9082be 100644
--- a/webscout/Provider/geminiapi.py
+++ b/webscout/Provider/geminiapi.py
@@ -5,19 +5,17 @@
"""
import json
-import os
-from typing import Any, Dict, Optional, Generator, Union, List
+from typing import Any, Dict, Generator, Optional, Union
-from curl_cffi.requests import Session
from curl_cffi import CurlError
+from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
from webscout.litagent import LitAgent
+
class GEMINIAPI(Provider):
"""
A class to interact with the Gemini API using OpenAI-compatible endpoint with LitAgent user-agent.
@@ -25,12 +23,12 @@ class GEMINIAPI(Provider):
required_auth = True
@classmethod
- def get_models(cls, api_key: str = None):
+ def get_models(cls, api_key: Optional[str] = None):
"""Fetch available models from Gemini API.
-
+
Args:
api_key (str, optional): Gemini API key
-
+
Returns:
list: List of available model IDs
"""
@@ -42,21 +40,21 @@ def get_models(cls, api_key: str = None):
headers = {
"Authorization": f"Bearer {api_key}",
}
-
+
response = temp_session.get(
"https://generativelanguage.googleapis.com/v1beta/openai/models",
headers=headers,
impersonate="chrome110"
)
-
+
if response.status_code != 200:
raise Exception(f"API request failed with status {response.status_code}: {response.text}")
-
+
data = response.json()
if "data" in data and isinstance(data["data"], list):
return [model["id"] for model in data["data"] if "id" in model]
raise Exception("Invalid response format from API")
-
+
except (CurlError, Exception) as e:
raise Exception(f"Failed to fetch models: {str(e)}")
@@ -78,12 +76,12 @@ def __init__(
top_p: float = 1,
model: str = "gemini-1.5-flash",
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
base_url: str = "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions",
system_prompt: str = "You are a helpful assistant.",
browser: str = "chrome"
@@ -173,9 +171,10 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Union[Dict[str, Any], Generator]:
+ **kwargs: Any,
+ ) -> Response:
conversation_prompt = self.conversation.gen_complete_prompt(prompt)
if optimizer:
if optimizer in self.__available_optimizers:
@@ -285,8 +284,9 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
+ **kwargs: Any,
) -> Union[str, Generator[str, None, None]]:
def for_stream_chat():
gen = self.ask(
diff --git a/webscout/Provider/julius.py b/webscout/Provider/julius.py
index 5a627374..1facf8ad 100644
--- a/webscout/Provider/julius.py
+++ b/webscout/Provider/julius.py
@@ -1,14 +1,13 @@
+import json
import uuid
+from typing import Any, Generator, Optional, Union
import requests
-import json
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream
-from webscout.AIbase import Provider
+
from webscout import exceptions
-from typing import Union, Any, Generator, Dict
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
class Julius(Provider):
@@ -37,12 +36,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 600,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
model: str = "Gemini Flash",
):
"""Instantiates Julius
@@ -57,7 +56,7 @@ def __init__(
proxies (dict, optional): Http request proxies. Defaults to {}.
history_offset (int, optional): Limit conversation history to this number of last texts. Defaults to 10250.
act (str|int, optional): Awesome prompt key or index. (Used as intro). Defaults to None.
- model (str, optional): Model to use for generating text. Defaults to "Gemini Flash".
+ model (str, optional): Model to use for generating text. Defaults to "Gemini Flash".
Options: "Llama 3", "GPT-4o", "GPT-3.5", "Command R", "Gemini Flash", "Gemini 1.5".
"""
if model not in self.AVAILABLE_MODELS:
@@ -108,9 +107,10 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> dict:
+ **kwargs: Any,
+ ) -> Response:
"""Chat with AI
Args:
@@ -133,7 +133,7 @@ def ask(
raise Exception(
f"Optimizer is not one of {self.__available_optimizers}"
)
-
+
payload = {
"message": {"content": conversation_prompt, "role": "user"},
"provider": "default",
@@ -190,7 +190,7 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
raw: bool = False,
) -> Union[str, Generator[str, None, None]]:
@@ -235,5 +235,8 @@ def get_message(self, response: dict) -> str:
from rich import print
ai = Julius(api_key="",timeout=5000)
response = ai.chat("write a poem about AI", stream=True)
- for chunk in response:
- print(chunk, end="", flush=True)
\ No newline at end of file
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)):
+ for chunk in response:
+ print(chunk, end="", flush=True)
+ else:
+ print(response)
diff --git a/webscout/Provider/learnfastai.py b/webscout/Provider/learnfastai.py
index da03473b..5eb9f8dd 100644
--- a/webscout/Provider/learnfastai.py
+++ b/webscout/Provider/learnfastai.py
@@ -1,15 +1,18 @@
-import os
import json
-from typing import Any, Dict, Optional, Union, Generator
+import os
import uuid
-from curl_cffi.requests import Session
+from typing import Any, Dict, Generator, Optional, Union
+
from curl_cffi import CurlError
+from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation, sanitize_stream # Import sanitize_stream
-from webscout.AIutel import AwesomePrompts
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import ( # Import sanitize_stream
+ AwesomePrompts,
+ Conversation,
+ Optimizers,
+)
class LearnFast(Provider):
@@ -22,12 +25,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 600, # Note: max_tokens is not used by this API
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
system_prompt: str = "You are a helpful AI assistant.", # Note: system_prompt is not used by this API
):
"""
@@ -105,10 +108,10 @@ def upload_image_to_0x0(self, image_path: str) -> str:
files = {"file": img_file}
try:
response = self.session.post(
- "https://0x0.st",
+ "https://0x0.st",
files=files,
# Add impersonate if using the main session
- impersonate="chrome110"
+ impersonate="chrome110"
)
response.raise_for_status()
image_url = response.text.strip()
@@ -148,12 +151,13 @@ def create_payload(
def ask(
self,
prompt: str,
- stream: bool = False, # API supports streaming
+ stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
image_path: Optional[str] = None,
- ) -> Union[dict, Generator[dict, None, None], str]:
+ **kwargs: Any,
+ ) -> Response:
"""Chat with LearnFast
Args:
@@ -205,10 +209,10 @@ def for_stream():
full_response = ""
try:
response = self.session.post(
- self.api_endpoint,
+ self.api_endpoint,
headers=current_headers, # Use headers with uniqueid
- data=data,
- stream=True,
+ data=data,
+ stream=True,
timeout=self.timeout,
impersonate="chrome110"
)
@@ -261,16 +265,17 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
image_path: Optional[str] = None,
- raw: bool = False
+ raw: bool = False,
+ **kwargs: Any,
) -> Union[str, Generator[str, None, None]]:
"""Generate response `str` or stream, with raw support"""
try:
response_gen = self.ask(
prompt, stream=stream, raw=raw,
- optimizer=optimizer, conversationally=conversationally,
+ optimizer=optimizer, conversationally=conversationally,
image_path=image_path
)
if stream:
@@ -306,5 +311,8 @@ def get_message(self, response: dict) -> str:
from rich import print
ai = LearnFast()
response = ai.chat("Hello, how are you?", stream=True, raw=False)
- for chunk in response:
- print(chunk, end='', flush=True)
\ No newline at end of file
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)):
+ for chunk in response:
+ print(chunk, end='', flush=True)
+ else:
+ print(response)
diff --git a/webscout/Provider/llama3mitril.py b/webscout/Provider/llama3mitril.py
index 99372bc4..06c09f0c 100644
--- a/webscout/Provider/llama3mitril.py
+++ b/webscout/Provider/llama3mitril.py
@@ -1,12 +1,12 @@
-from curl_cffi.requests import Session
-from curl_cffi import CurlError
import json
-from typing import Union, Any, Dict, Generator
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts
-from webscout.AIbase import Provider
+from typing import Any, Dict, Generator, Optional, Union
+
+from curl_cffi import CurlError
+from curl_cffi.requests import Session
+
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers
class Llama3Mitril(Provider):
@@ -19,12 +19,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 2048,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
system_prompt: str = "You are a helpful, respectful and honest assistant.",
temperature: float = 0.8,
):
@@ -76,9 +76,10 @@ def ask(
prompt: str,
stream: bool = True, # API supports streaming
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Union[Dict[str, Any], Generator[Any, None, None]]:
+ **kwargs: Any,
+ ) -> Response:
"""Sends a prompt to the Llama3 Mitril API and returns the response."""
conversation_prompt = self.conversation.gen_complete_prompt(prompt)
if optimizer:
@@ -129,7 +130,7 @@ def for_stream():
resp = {"text": token_text}
# Yield dict or raw string chunk
yield resp if not raw else token_text
- except (json.JSONDecodeError, IndexError, UnicodeDecodeError) as e:
+ except (json.JSONDecodeError, IndexError, UnicodeDecodeError):
# Ignore errors in parsing specific lines
continue
@@ -171,8 +172,9 @@ def chat(
self,
prompt: str,
stream: bool = True, # Default to True as API supports it
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
+ **kwargs: Any,
) -> Union[str, Generator[str, None, None]]:
"""Generates a response from the Llama3 Mitril API."""
@@ -212,4 +214,4 @@ def get_message(self, response: Dict[str, Any]) -> str:
)
for response in ai.chat("Hello", stream=True):
- print(response, end="", flush=True)
\ No newline at end of file
+ print(response, end="", flush=True)
diff --git a/webscout/Provider/llmchat.py b/webscout/Provider/llmchat.py
index 979c4fdb..febe51c6 100644
--- a/webscout/Provider/llmchat.py
+++ b/webscout/Provider/llmchat.py
@@ -1,14 +1,12 @@
-from curl_cffi.requests import Session
+from typing import Any, Dict, Generator, Union
+
from curl_cffi import CurlError
-import json
-from typing import Union, Any, Dict, Optional, Generator, List
+from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers, sanitize_stream
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts
-from webscout.AIbase import Provider
from webscout import exceptions
-from webscout.litagent import LitAgent as Lit
+from webscout.AIbase import Provider
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
+
class LLMChat(Provider):
"""
@@ -95,7 +93,7 @@ def __init__(
self.last_response = {}
self.model = model
self.system_prompt = system_prompt
-
+
self.headers = {
"Content-Type": "application/json",
"Accept": "*/*",
@@ -161,9 +159,9 @@ def for_stream():
full_response = ""
try:
response = self.session.post(
- url,
- json=payload,
- stream=True,
+ url,
+ json=payload,
+ stream=True,
timeout=self.timeout,
impersonate="chrome110"
)
@@ -252,11 +250,11 @@ def get_message(self, response: Dict[str, Any]) -> str:
print("-" * 80)
print(f"{'Model':<50} {'Status':<10} {'Response'}")
print("-" * 80)
-
+
# Test all available models
working = 0
total = len(LLMChat.AVAILABLE_MODELS)
-
+
for model in LLMChat.AVAILABLE_MODELS:
try:
test_ai = LLMChat(model=model, timeout=60)
@@ -265,7 +263,7 @@ def get_message(self, response: Dict[str, Any]) -> str:
for chunk in response:
response_text += chunk
print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True)
-
+
if response_text and len(response_text.strip()) > 0:
status = "✓"
# Truncate response if too long
@@ -279,4 +277,4 @@ def get_message(self, response: Dict[str, Any]) -> str:
# ai = LLMChat(model="@cf/meta/llama-3.1-70b-instruct")
# response = ai.chat("Say 'Hello' in one word", stream=True, raw=False)
# for chunk in response:
- # print(chunk, end="", flush=True)
\ No newline at end of file
+ # print(chunk, end="", flush=True)
diff --git a/webscout/Provider/llmchatco.py b/webscout/Provider/llmchatco.py
index 6dce6e65..1b3a33a1 100644
--- a/webscout/Provider/llmchatco.py
+++ b/webscout/Provider/llmchatco.py
@@ -1,17 +1,20 @@
-from curl_cffi.requests import Session
-from curl_cffi import CurlError
-import json
import uuid
-import re
-from typing import Union, Any, Dict, Optional, Generator, List
+from typing import Any, Dict, Generator, Optional, Union
+
+from curl_cffi import CurlError
+from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers, sanitize_stream # Import sanitize_stream
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider
+from webscout.AIutel import ( # Import sanitize_stream
+ AwesomePrompts,
+ Conversation,
+ Optimizers,
+ sanitize_stream,
+)
from webscout.litagent import LitAgent as Lit
+
class LLMChatCo(Provider):
"""
A class to interact with the LLMChat.co API
@@ -288,4 +291,4 @@ def get_message(self, response: Dict[str, Any]) -> str:
ai = LLMChatCo()
response = ai.chat("yooo", stream=True, raw=False)
for chunk in response:
- print(chunk, end="", flush=True)
\ No newline at end of file
+ print(chunk, end="", flush=True)
diff --git a/webscout/Provider/meta.py b/webscout/Provider/meta.py
index 1b33f125..e27887e5 100644
--- a/webscout/Provider/meta.py
+++ b/webscout/Provider/meta.py
@@ -1,22 +1,20 @@
import json
+import random
import time
import urllib
import uuid
from typing import Dict, Generator, List, Union
-import random
from curl_cffi import CurlError
from curl_cffi.requests import Session
-from webscout.scout import Scout
+from litprinter import ic
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts
-from webscout.AIutel import retry
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, retry
from webscout.litagent import LitAgent as Lit
-from litprinter import ic
+from webscout.scout import Scout
+
MAX_RETRIES = 3
HTTP2_STREAM_ERRORS = [92, 18, 7, 35, 36] # Common curl HTTP/2 stream errors
@@ -266,7 +264,8 @@ def get_fb_session(email, password, proxies=None):
"Was not able to login to Facebook. Please check your credentials. "
"You may also have been rate limited. Try to connect to Facebook manually."
)
- ic.configureOutput(prefix='INFO| '); ic("Successfully logged in to Facebook.")
+ ic.configureOutput(prefix='INFO| ')
+ ic("Successfully logged in to Facebook.")
return cookies
@@ -348,7 +347,7 @@ def __init__(
"user-agent": Lit().random(),
}
)
-
+
# Configure session for better HTTP/2 handling
self.session.timeout = timeout
self.session.curl_options.update({
@@ -364,18 +363,18 @@ def __init__(
64: 0, # CURLOPT_SSL_VERIFYPEER
81: 0, # CURLOPT_SSL_VERIFYHOST
})
-
+
# Create a backup session for fallback
self.backup_session = Session()
self.backup_session.headers.update({"user-agent": Lit().random()})
self.backup_session.curl_options.update(self.session.curl_options)
-
+
# Add HTTP/2 error tracking
self.http2_error_count = 0
self.max_http2_errors = 3
self.last_successful_request = time.time()
self.retry_count = 0
-
+
self.access_token = None
self.fb_email = fb_email
self.fb_password = fb_password
@@ -414,7 +413,7 @@ def __init__(
self.session.proxies = proxies
# If skip_init was True we won't have cookies yet — some methods will fetch them lazily
if self.skip_init:
- ic.configureOutput(prefix='WARNING| ');
+ ic.configureOutput(prefix='WARNING| ')
ic('Meta initialized in skip_init mode: cookies not fetched. Some operations will fail until cookies are obtained.')
@@ -486,7 +485,7 @@ def get_access_token(self) -> str:
# Some Curl errors are wrapped in requests.HTTPError from curl_cffi; inspect message
err_str = str(e)
if 'HTTP/2 stream' in err_str or 'stream' in err_str:
- ic.configureOutput(prefix='WARNING| ');
+ ic.configureOutput(prefix='WARNING| ')
ic(f"Detected HTTP/2 stream issue when getting access token: {err_str}. Attempting HTTP/1.1 fallback via requests")
try:
import requests
@@ -615,7 +614,7 @@ def for_stream():
except CurlError as e:
# Try HTTP/1.1 fallback once
if hasattr(e, 'errno') and e.errno in HTTP2_STREAM_ERRORS:
- ic.configureOutput(prefix='WARNING| ');
+ ic.configureOutput(prefix='WARNING| ')
ic("HTTP/2 stream error on streaming request, attempting HTTP/1.1 fallback")
try:
self.session.curl_options.update({84: 1}) # force HTTP/1.1
@@ -654,7 +653,7 @@ def for_stream():
except CurlError as e:
# Handle HTTP/2 stream closure during iteration
if hasattr(e, 'errno') and e.errno in HTTP2_STREAM_ERRORS:
- ic.configureOutput(prefix='WARNING| ');
+ ic.configureOutput(prefix='WARNING| ')
ic(f"HTTP/2 stream closed during iteration (errno: {e.errno})")
if final_message:
# Yield the last complete message before the stream closed
@@ -663,7 +662,7 @@ def for_stream():
else:
raise
except (ConnectionError, TimeoutError) as e:
- ic.configureOutput(prefix='WARNING| ');
+ ic.configureOutput(prefix='WARNING| ')
ic(f"Connection error during streaming: {e}")
if final_message:
# Yield the last complete message before the connection was lost
@@ -672,12 +671,12 @@ def for_stream():
prompt, self.get_message(self.last_response)
)
return
-
+
if final_message:
self.conversation.update_chat_history(
prompt, self.get_message(self.last_response)
)
-
+
except CurlError as e:
if hasattr(e, 'errno') and e.errno in HTTP2_STREAM_ERRORS:
raise exceptions.FailedToGenerateResponseError(
@@ -697,7 +696,7 @@ def for_stream():
except CurlError as e:
# Try HTTP/1.1 fallback for non-stream requests
if hasattr(e, 'errno') and e.errno in HTTP2_STREAM_ERRORS:
- ic.configureOutput(prefix='WARNING| ');
+ ic.configureOutput(prefix='WARNING| ')
ic("HTTP/2 error on non-stream request, attempting HTTP/1.1 fallback")
try:
self.session.curl_options.update({84: 1}) # force HTTP/1.1
@@ -873,7 +872,7 @@ def get_cookies(self) -> dict:
last_response = response
break
except Exception as e:
- ic.configureOutput(prefix='WARNING| ');
+ ic.configureOutput(prefix='WARNING| ')
ic(f"Attempt {attempt+1} to fetch meta.ai failed: {e}. Retrying...")
time.sleep(1 * (2 ** attempt))
if last_response is None:
@@ -983,10 +982,10 @@ def get_message(self, response: dict) -> str:
for chunk in ai:
print(chunk, end="", flush=True)
except exceptions.FailedToGenerateResponseError as e:
- ic.configureOutput(prefix='ERROR| ');
+ ic.configureOutput(prefix='ERROR| ')
ic(f"Meta provider failed to initialize or run: {e}")
ic("Possible causes: network connectivity issues, region blocking, or site returning error pages.")
ic("For offline testing, re-run with: Meta(skip_init=True)")
except Exception as e:
- ic.configureOutput(prefix='ERROR| ');
+ ic.configureOutput(prefix='ERROR| ')
ic(f"Unexpected error running meta provider: {e}")
diff --git a/webscout/Provider/oivscode.py b/webscout/Provider/oivscode.py
index 7ba3533f..7fcfdaf9 100644
--- a/webscout/Provider/oivscode.py
+++ b/webscout/Provider/oivscode.py
@@ -1,10 +1,11 @@
import random
-import requests
import secrets
import string
import uuid
from typing import Any, Dict, Optional, Union
+import requests
+
from webscout import exceptions
from webscout.AIbase import Provider
from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
diff --git a/webscout/Provider/searchchat.py b/webscout/Provider/searchchat.py
index e0b5bad5..5cd90214 100644
--- a/webscout/Provider/searchchat.py
+++ b/webscout/Provider/searchchat.py
@@ -1,16 +1,20 @@
-from curl_cffi.requests import Session
+from datetime import datetime, timezone
+from typing import Any, Dict, Generator, Optional, Union
+
from curl_cffi import CurlError
-import json
-from datetime import datetime
-from typing import Any, Dict, Optional, Generator, Union
+from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import ( # Import sanitize_stream
+ AwesomePrompts,
+ Conversation,
+ Optimizers,
+ sanitize_stream,
+)
from webscout.litagent import LitAgent
+
class SearchChatAI(Provider):
"""
A class to interact with the SearchChatAI API.
@@ -22,12 +26,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 2049,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
):
"""Initializes the SearchChatAI API client."""
self.url = "https://search-chat.ai/api/chat-test-stop.php"
@@ -35,12 +39,12 @@ def __init__(
self.is_conversation = is_conversation
self.max_tokens_to_sample = max_tokens
self.last_response = {}
-
+
# Initialize LitAgent for user agent generation
self.agent = LitAgent()
# Use fingerprinting to create a consistent browser identity
self.fingerprint = self.agent.generate_fingerprint("chrome")
-
+
# Use the fingerprint for headers
self.headers = {
"Accept": self.fingerprint["accept"],
@@ -57,7 +61,7 @@ def __init__(
"Sec-CH-UA-Platform": f'"{self.fingerprint["platform"]}"',
"User-Agent": self.fingerprint["user_agent"],
}
-
+
# Initialize curl_cffi Session
self.session = Session()
# Update curl_cffi session headers and proxies
@@ -85,13 +89,13 @@ def __init__(
def refresh_identity(self, browser: str = None):
"""
Refreshes the browser identity fingerprint.
-
+
Args:
browser: Specific browser to use for the new fingerprint
"""
browser = browser or self.fingerprint.get("browser_type", "chrome")
self.fingerprint = self.agent.generate_fingerprint(browser)
-
+
# Update headers with new fingerprint
self.headers.update({
"Accept": self.fingerprint["accept"],
@@ -100,11 +104,11 @@ def refresh_identity(self, browser: str = None):
"Sec-CH-UA-Platform": f'"{self.fingerprint["platform"]}"',
"User-Agent": self.fingerprint["user_agent"],
})
-
+
# Update session headers (already done in the original code, should work with curl_cffi session)
for header, value in self.headers.items():
self.session.headers[header] = value
-
+
return self.fingerprint
def ask(
@@ -112,19 +116,20 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Union[Dict[str, Any], Generator]:
+ **kwargs: Any,
+ ) -> Response:
"""
Send a message to the API and get the response.
-
+
Args:
prompt: The message to send
stream: Whether to stream the response
raw: Whether to return raw response
optimizer: The optimizer to use
conversationally: Whether to use conversation history
-
+
Returns:
Either a dictionary with the response or a generator for streaming
"""
@@ -147,7 +152,7 @@ def ask(
"text": conversation_prompt
}
],
- "timestamp": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+ "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
}
]
}
@@ -182,7 +187,7 @@ def for_stream():
raise exceptions.FailedToGenerateResponseError(
f"Request failed with status code {response.status_code} - {response.text}"
)
-
+
streaming_text = ""
# Use sanitize_stream
processed_stream = sanitize_stream(
@@ -197,12 +202,12 @@ def for_stream():
for content_chunk in processed_stream:
if raw:
- yield content_chunk
+ yield content_chunk
else:
if content_chunk and isinstance(content_chunk, str):
streaming_text += content_chunk
yield dict(text=content_chunk)
-
+
# Update history and last response after stream finishes
self.last_response = {"text": streaming_text}
self.conversation.update_chat_history(prompt, streaming_text)
@@ -216,13 +221,13 @@ def for_non_stream():
full_text = ""
# Iterate through the generator provided by for_stream
# Ensure raw=False so for_stream yields dicts
- for chunk_data in for_stream():
+ for chunk_data in for_stream():
if isinstance(chunk_data, dict) and "text" in chunk_data:
full_text += chunk_data["text"]
# If raw=True was somehow passed, handle string chunks
elif isinstance(chunk_data, str):
full_text += chunk_data
-
+
# last_response and history are updated within for_stream
# Return the final aggregated response dict or raw string
return full_text if raw else self.last_response
@@ -234,19 +239,20 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
raw: bool = False, # Added raw parameter
+ **kwargs: Any,
) -> Union[str, Generator[str, None, None]]:
"""
Chat with the API.
-
+
Args:
prompt: The message to send
stream: Whether to stream the response
optimizer: The optimizer to use
conversationally: Whether to use conversation history
-
+
Returns:
Either a string response or a generator for streaming
"""
@@ -268,13 +274,17 @@ def for_non_stream_chat():
return self.get_message(response_data)
return for_stream_chat() if stream else for_non_stream_chat()
- def get_message(self, response: dict) -> str:
+ def get_message(self, response: Response) -> str:
"""Extract the message from the response."""
- assert isinstance(response, dict), "Response should be of dict data-type only"
- return response["text"]
+ if not isinstance(response, dict):
+ return str(response)
+ return response.get("text", "")
if __name__ == "__main__":
ai = SearchChatAI()
resp = ai.chat("Hello", stream=True, raw=True)
- for chunk in resp:
- print(chunk, end="")
\ No newline at end of file
+ if hasattr(resp, "__iter__") and not isinstance(resp, (str, bytes)):
+ for chunk in resp:
+ print(chunk, end="")
+ else:
+ print(resp)
diff --git a/webscout/Provider/sonus.py b/webscout/Provider/sonus.py
index 7cfeedeb..fc783d94 100644
--- a/webscout/Provider/sonus.py
+++ b/webscout/Provider/sonus.py
@@ -1,13 +1,19 @@
-from curl_cffi.requests import Session
-from curl_cffi import CurlError
import json
-from typing import Any, Dict, Optional, Generator, Union
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream
-from webscout.AIbase import Provider
+from typing import Any, Dict, Generator, Optional, Union
+
+from curl_cffi import CurlError
+from curl_cffi.requests import Session
+
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import ( # Import sanitize_stream
+ AwesomePrompts,
+ Conversation,
+ Optimizers,
+)
from webscout.litagent import LitAgent
+
+
class SonusAI(Provider):
"""
A class to interact with the Sonus AI chat API.
@@ -24,20 +30,20 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 2049, # Note: max_tokens is not directly used by this API
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
model: str = "pro"
):
"""Initializes the Sonus AI API client."""
if model not in self.AVAILABLE_MODELS:
raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}")
-
+
self.url = "https://chat.sonus.ai/chat.php"
-
+
# Headers for the request
self.headers = {
'Accept': '*/*',
@@ -47,7 +53,7 @@ def __init__(
'User-Agent': LitAgent().random()
# Add sec-ch-ua headers if needed for impersonation consistency
}
-
+
# Initialize curl_cffi Session
self.session = Session()
# Update curl_cffi session headers and proxies
@@ -90,10 +96,11 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
reasoning: bool = False,
- ) -> Union[Dict[str, Any], Generator]:
+ **kwargs: Any,
+ ) -> Response:
conversation_prompt = self.conversation.gen_complete_prompt(prompt)
if optimizer:
if optimizer in self.__available_optimizers:
@@ -119,9 +126,9 @@ def for_stream():
try:
# Use curl_cffi session post with impersonate
response = self.session.post(
- self.url,
+ self.url,
data=form_data,
- stream=True,
+ stream=True,
timeout=self.timeout,
impersonate="chrome110"
)
@@ -151,11 +158,11 @@ def for_stream():
yield dict(text=content)
except json.JSONDecodeError:
continue
-
+
# Update history and last response after stream finishes
self.last_response = {"text": streaming_text}
self.conversation.update_chat_history(prompt, streaming_text)
-
+
except CurlError as e:
raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {str(e)}") from e
except Exception as e:
@@ -165,7 +172,7 @@ def for_non_stream():
try:
# Use curl_cffi session post with impersonate
response = self.session.post(
- self.url,
+ self.url,
# headers are set on the session
data=form_data, # Use data for multipart form fields
timeout=self.timeout,
@@ -196,7 +203,7 @@ def for_non_stream():
self.conversation.update_chat_history(prompt, full_response)
# Return dict or raw string
return full_response if raw else {"text": full_response}
-
+
except CurlError as e: # Catch CurlError
raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {str(e)}") from e
except Exception as e: # Catch other potential exceptions
@@ -208,10 +215,11 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
reasoning: bool = False,
raw: bool = False, # Added raw parameter
+ **kwargs: Any,
) -> Union[str, Generator[str, None, None]]:
def for_stream_chat():
for response in self.ask(
@@ -231,12 +239,16 @@ def for_non_stream_chat():
return self.get_message(response_data)
return for_stream_chat() if stream else for_non_stream_chat()
- def get_message(self, response: dict) -> str:
- assert isinstance(response, dict), "Response should be of dict data-type only"
- return response["text"]
+ def get_message(self, response: Response) -> str:
+ if not isinstance(response, dict):
+ return str(response)
+ return response.get("text", "")
if __name__ == "__main__":
sonus = SonusAI()
resp = sonus.chat("Hello", stream=True, raw=True)
- for chunk in resp:
- print(chunk, end="")
\ No newline at end of file
+ if hasattr(resp, "__iter__") and not isinstance(resp, (str, bytes)):
+ for chunk in resp:
+ print(chunk, end="")
+ else:
+ print(resp)
diff --git a/webscout/Provider/toolbaz.py b/webscout/Provider/toolbaz.py
index 5ff5ef95..3b52e49a 100644
--- a/webscout/Provider/toolbaz.py
+++ b/webscout/Provider/toolbaz.py
@@ -1,20 +1,25 @@
-import re
-from curl_cffi.requests import Session
-from curl_cffi import CurlError
-import uuid
import base64
import json
import random
+import re
import string
import time
+import uuid
from datetime import datetime
-from typing import Any, Dict, Optional, Generator, Union, List
+from typing import Any, Dict, Generator, Optional, Union
+
+from curl_cffi import CurlError
+from curl_cffi.requests import Session
from webscout import exceptions
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream
-from webscout.AIbase import Provider
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import ( # Import sanitize_stream
+ AwesomePrompts,
+ Conversation,
+ Optimizers,
+ sanitize_stream,
+)
+
class Toolbaz(Provider):
"""
@@ -37,7 +42,7 @@ class Toolbaz(Provider):
"grok-4-fast",
"grok-4.1-fast",
-
+
"toolbaz-v4.5-fast",
"toolbaz_v4",
"toolbaz_v3.5_pro",
@@ -48,7 +53,7 @@ class Toolbaz(Provider):
"Llama-4-Maverick",
"Llama-3.3-70B",
-
+
"mixtral_8x22b",
"L3-70B-Euryale-v2.1",
"midnight-rose",
@@ -60,12 +65,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 600, # Note: max_tokens is not directly used by the API
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
model: str = "gemini-2.0-flash",
system_prompt: str = "You are a helpful AI assistant." # Note: system_prompt is not directly used by the API
):
@@ -156,7 +161,7 @@ def get_auth(self):
}
# Use curl_cffi session post WITHOUT impersonate for token request
resp = self.session.post(
- "https://data.toolbaz.com/token.php",
+ "https://data.toolbaz.com/token.php",
data=data
# Removed impersonate="chrome110" for this specific request
)
@@ -181,10 +186,11 @@ def ask(
self,
prompt: str,
stream: bool = False,
- raw: bool = False, # Kept for compatibility, but output is always dict/string
+ raw: bool = False,
optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Union[Dict[str, Any], Generator]:
+ **kwargs: Any,
+ ) -> Response:
"""Sends a prompt to the Toolbaz API and returns the response."""
if optimizer and optimizer not in self.__available_optimizers:
raise exceptions.FailedToGenerateResponseError(f"Optimizer is not one of {self.__available_optimizers}")
@@ -196,7 +202,7 @@ def ask(
)
# get_auth now raises exceptions on failure
- auth = self.get_auth()
+ auth = self.get_auth()
# No need to check if auth is None, as an exception would have been raised
data = {
@@ -264,7 +270,7 @@ def for_non_stream():
resp.raise_for_status()
# Use response.text which is already decoded
- text = resp.text
+ text = resp.text
# Remove [model: ...] tags
text = re.sub(r"\[model:.*?\]", "", text)
@@ -287,6 +293,7 @@ def chat(
optimizer: Optional[str] = None,
conversationally: bool = False,
raw: bool = False, # Added raw parameter
+ **kwargs: Any,
) -> Union[str, Generator[str, None, None]]:
"""Generates a response from the Toolbaz API."""
def for_stream_chat():
@@ -319,7 +326,7 @@ def for_non_stream_chat():
return for_stream_chat() if stream else for_non_stream_chat()
- def get_message(self, response: Dict[str, Any]) -> str:
+ def get_message(self, response: Response) -> str:
"""Extract the message from the response.
Args:
@@ -328,13 +335,14 @@ def get_message(self, response: Dict[str, Any]) -> str:
Returns:
str: Message extracted
"""
- assert isinstance(response, dict), "Response should be of dict data-type only"
+ if not isinstance(response, dict):
+ return str(response)
return response.get("text", "")
# Example usage
if __name__ == "__main__":
# Ensure curl_cffi is installed
- from rich import print # Use rich print if available
+ from rich import print # Use rich print if available
print("-" * 80)
print(f"{'Model':<50} {'Status':<10} {'Response'}")
print("-" * 80)
@@ -346,10 +354,11 @@ def get_message(self, response: Dict[str, Any]) -> str:
response_stream = test_ai.chat("Say 'Hello' in one word", stream=True)
response_text = ""
# print(f"\r{model:<50} {'Streaming...':<10}", end="", flush=True)
- for chunk in response_stream:
- response_text += chunk
- # Optional: print chunks for visual feedback
- # print(chunk, end="", flush=True)
+ if hasattr(response_stream, "__iter__") and not isinstance(response_stream, (str, bytes)):
+ for chunk in response_stream:
+ response_text += chunk
+ else:
+ response_text = str(response_stream)
if response_text and len(response_text.strip()) > 0:
status = "✓"
@@ -369,4 +378,4 @@ def get_message(self, response: Dict[str, Any]) -> str:
except Exception as e:
# Print full error for debugging
- print(f"\r{model:<50} {'✗':<10} Error: {str(e)}")
\ No newline at end of file
+ print(f"\r{model:<50} {'✗':<10} Error: {str(e)}")
diff --git a/webscout/Provider/turboseek.py b/webscout/Provider/turboseek.py
index 0fe1cb5f..cf71f6b2 100644
--- a/webscout/Provider/turboseek.py
+++ b/webscout/Provider/turboseek.py
@@ -1,16 +1,21 @@
import re
-from typing import Optional, Union, Any, AsyncGenerator, Dict
-from curl_cffi.requests import Session
+from typing import Any, Generator, Optional, Union
+
from curl_cffi import CurlError
+from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import ( # Import sanitize_stream
+ AwesomePrompts,
+ Conversation,
+ Optimizers,
+ sanitize_stream,
+)
from webscout.litagent import LitAgent
+
class TurboSeek(Provider):
"""
This class provides methods for interacting with the TurboSeek API.
@@ -23,12 +28,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 600,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
model: str = "Llama 3.1 70B" # Note: model parameter is not used by the API endpoint
):
"""Instantiates TurboSeek
@@ -96,34 +101,34 @@ def _html_to_markdown(text: str) -> str:
"""Convert basic HTML tags to Markdown."""
if not text:
return ""
-
+
# Unescape HTML entities first
import html
text = html.unescape(text)
# Headers
text = re.sub(r']*>(.*?)', r'\n# \1\n', text)
-
+
# Lists
text = re.sub(r']*>(.*?)', r'\n* \1', text)
text = re.sub(r'<(ul|ol)[^>]*>', r'\n', text)
text = re.sub(r'(ul|ol)>', r'\n', text)
-
+
# Paragraphs and Breaks
text = re.sub(r'', r'\n\n', text)
text = re.sub(r']*>', r'\n', text)
text = re.sub(r'
', r'\n', text)
-
+
# Bold and Italic
text = re.sub(r'<(strong|b)[^>]*>(.*?)\1>', r'**\2**', text)
text = re.sub(r'<(em|i)[^>]*>(.*?)\1>', r'*\2*', text)
-
+
# Remove structural tags
text = re.sub(r'?(section|div|span|article|header|footer)[^>]*>', '', text, flags=re.IGNORECASE)
-
+
# Final cleanup of remaining tags
text = re.sub(r'<[^>]*>', '', text)
-
+
return text
@staticmethod
@@ -139,9 +144,10 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> dict:
+ **kwargs: Any,
+ ) -> Response:
"""Chat with AI
"""
conversation_prompt = self.conversation.gen_complete_prompt(prompt)
@@ -163,9 +169,9 @@ def ask(
def for_stream():
try:
response = self.session.post(
- self.chat_endpoint,
- json=payload,
- stream=True,
+ self.chat_endpoint,
+ json=payload,
+ stream=True,
timeout=self.timeout,
impersonate="chrome120"
)
@@ -173,23 +179,23 @@ def for_stream():
raise exceptions.FailedToGenerateResponseError(
f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}"
)
-
+
streaming_text = ""
# The API returns raw HTML chunks now, no "data:" prefix
processed_stream = sanitize_stream(
data=response.iter_content(chunk_size=None),
- intro_value=None,
+ intro_value=None,
to_json=False,
strip_chars='', # Disable default lstrip to preserve spacing
content_extractor=self._turboseek_extractor,
yield_raw_on_error=True,
raw=raw
)
-
+
for content_chunk in processed_stream:
if content_chunk is None:
continue
-
+
if raw:
yield content_chunk
else:
@@ -202,7 +208,7 @@ def for_stream():
streaming_text += clean_chunk
self.last_response.update(dict(text=streaming_text))
yield dict(text=clean_chunk)
-
+
if not raw and streaming_text:
self.conversation.update_chat_history(
prompt, streaming_text
@@ -219,15 +225,15 @@ def for_non_stream():
# We use ask(..., raw=True) internally or just the local for_stream
# Actually, let's just make a sub-call
response = self.session.post(
- self.chat_endpoint,
- json=payload,
+ self.chat_endpoint,
+ json=payload,
timeout=self.timeout,
impersonate="chrome120"
)
full_html = response.text
except Exception as e:
raise exceptions.FailedToGenerateResponseError(f"Failed to get non-stream response: {e}") from e
-
+
# Convert full HTML to Markdown
final_text = self._html_to_markdown(full_html).strip()
self.last_response = {"text": final_text}
@@ -239,10 +245,11 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
raw: bool = False, # Added raw parameter
- ) -> str:
+ **kwargs: Any,
+ ) -> Union[str, Generator[str, None, None]]:
"""Generate response `str`
Args:
prompt (str): Prompt to be send.
@@ -275,23 +282,24 @@ def for_non_stream():
return self.get_message(result)
return for_stream() if stream else for_non_stream()
- def get_message(self, response: dict) -> str:
+ def get_message(self, response: Response) -> str:
"""Retrieves message only from response
Args:
- response (dict): Response generated by `self.ask`
+ response (Response): Response generated by `self.ask`
Returns:
str: Message extracted
"""
- assert isinstance(response, dict), "Response should be of dict data-type only"
+ if not isinstance(response, dict):
+ return str(response)
# Unicode escapes are handled by json.loads within sanitize_stream
- return response.get("text", "")
+ return response.get("text", "")
if __name__ == '__main__':
import sys
ai = TurboSeek(timeout=60)
-
+
# helper for safe printing on windows
def safe_print(text, end="\n"):
try:
@@ -302,9 +310,16 @@ def safe_print(text, end="\n"):
safe_print("\n=== Testing Non-Streaming ===")
response = ai.chat("How can I get a 6 pack in 3 months?", stream=False)
- safe_print(response)
-
+ if isinstance(response, str):
+ safe_print(response)
+ else:
+ safe_print(str(response))
+
safe_print("\n=== Testing Streaming ===")
- for chunk in ai.chat("How can I get a 6 pack in 3 months?", stream=True):
- safe_print(chunk, end="")
- safe_print("")
\ No newline at end of file
+ stream_resp = ai.chat("How can I get a 6 pack in 3 months?", stream=True)
+ if hasattr(stream_resp, "__iter__") and not isinstance(stream_resp, (str, bytes)):
+ for chunk in stream_resp:
+ safe_print(chunk, end="")
+ else:
+ safe_print(str(stream_resp))
+ safe_print("")
diff --git a/webscout/Provider/typefully.py b/webscout/Provider/typefully.py
index 2265beb3..467e1856 100644
--- a/webscout/Provider/typefully.py
+++ b/webscout/Provider/typefully.py
@@ -1,13 +1,12 @@
-from typing import Any, Dict
+from typing import Any, Dict, Generator, Optional, Union
+
+from curl_cffi import CurlError
+from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream
from webscout.litagent import LitAgent
-from curl_cffi.requests import Session
-from curl_cffi import CurlError
class TypefullyAI(Provider):
@@ -24,12 +23,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 600,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
system_prompt: str = "You're a helpful assistant.",
model: str = "openai:gpt-4o-mini",
):
@@ -77,7 +76,6 @@ def __init__(
@staticmethod
def _typefully_extractor(chunk) -> str:
"""Extracts content from Typefully AI SSE format."""
- import re
import json
# Handle parsed JSON objects (when to_json=True)
@@ -120,9 +118,10 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Dict[str, Any]:
+ **kwargs: Any,
+ ) -> Response:
conversation_prompt = self.conversation.gen_complete_prompt(prompt)
if optimizer:
if optimizer in self.__available_optimizers:
@@ -191,10 +190,11 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
raw: bool = False,
- ) -> str:
+ **kwargs: Any,
+ ) -> Union[str, Generator[str, None, None]]:
def for_stream():
for response in self.ask(
prompt, True, raw=raw, optimizer=optimizer, conversationally=conversationally
@@ -219,8 +219,9 @@ def for_non_stream():
return for_stream() if stream else for_non_stream()
- def get_message(self, response: dict) -> str:
- assert isinstance(response, dict), "Response should be of dict data-type only"
+ def get_message(self, response: Response) -> str:
+ if not isinstance(response, dict):
+ return str(response)
text = response.get("text", "")
try:
formatted_text = text.replace("\\n", "\n").replace("\\n\\n", "\n\n")
@@ -240,8 +241,12 @@ def get_message(self, response: dict) -> str:
test_ai = TypefullyAI(model=model, timeout=60)
response_stream = test_ai.chat("Say 'Hello' in one word", stream=True)
response_text = ""
- for chunk in response_stream:
- response_text += chunk
+ if hasattr(response_stream, "__iter__") and not isinstance(response_stream, (str, bytes)):
+ for chunk in response_stream:
+ response_text += chunk
+ else:
+ response_text = str(response_stream)
+
if response_text and len(response_text.strip()) > 0:
status = "OK"
clean_text = response_text.strip()
diff --git a/webscout/Provider/x0gpt.py b/webscout/Provider/x0gpt.py
index 47982499..4bcf252e 100644
--- a/webscout/Provider/x0gpt.py
+++ b/webscout/Provider/x0gpt.py
@@ -1,16 +1,22 @@
-from typing import Generator, Union, Any, Dict
+from typing import Any, Dict, Generator, Optional, Union
from uuid import uuid4
+
from curl_cffi import CurlError
+
+# Import HTTPVersion enum
+from curl_cffi.const import CurlHttpVersion
from curl_cffi.requests import Session
-from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import ( # Import sanitize_stream
+ AwesomePrompts,
+ Conversation,
+ Optimizers,
+ sanitize_stream,
+)
from webscout.litagent import LitAgent
-# Import HTTPVersion enum
-from curl_cffi.const import CurlHttpVersion
+
class X0GPT(Provider):
"""
@@ -34,12 +40,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 600,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
system_prompt: str = "You are a helpful assistant.",
model: str = "UNKNOWN"
):
@@ -64,7 +70,7 @@ def __init__(
'You are a friendly assistant.'
"""
# Initialize curl_cffi Session instead of requests.Session
- self.session = Session()
+ self.session = Session()
self.is_conversation = is_conversation
self.max_tokens_to_sample = max_tokens
self.api_endpoint = "https://x0-gpt.devwtf.in/api/stream/reply"
@@ -121,9 +127,10 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Union[Dict[str, Any], Generator]:
+ **kwargs: Any,
+ ) -> Response:
"""
Sends a prompt to the x0-gpt.devwtf.in API and returns the response.
@@ -179,7 +186,7 @@ def for_stream():
raise exceptions.FailedToGenerateResponseError(
f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}"
)
-
+
streaming_response = ""
# Use sanitize_stream with regex-based extraction and filtering
processed_stream = sanitize_stream(
@@ -204,7 +211,7 @@ def for_stream():
# Always yield as string, even in raw mode
if isinstance(content_chunk, bytes):
content_chunk = content_chunk.decode('utf-8', errors='ignore')
-
+
if raw:
yield content_chunk
else:
@@ -246,9 +253,10 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
raw: bool = False, # Added raw parameter
+ **kwargs: Any,
) -> Union[str, Generator[str, None, None]]:
"""
Generates a response from the X0GPT API.
@@ -321,5 +329,8 @@ def get_message(self, response: dict) -> str:
from rich import print
ai = X0GPT(timeout=5000)
response = ai.chat("write a poem about AI", stream=True, raw=False)
- for chunk in response:
- print(chunk, end="", flush=True)
\ No newline at end of file
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)):
+ for chunk in response:
+ print(chunk, end="", flush=True)
+ else:
+ print(response)
diff --git a/webscout/Provider/yep.py b/webscout/Provider/yep.py
index ebf8148d..d7e5fc1a 100644
--- a/webscout/Provider/yep.py
+++ b/webscout/Provider/yep.py
@@ -1,14 +1,14 @@
import uuid
+from typing import Any, Dict, Generator, Optional, TypeVar, Union
+
from curl_cffi import CurlError
from curl_cffi.requests import Session
-from typing import Any, Dict, Optional, Generator, Union, List, TypeVar
-from webscout.AIutel import Optimizers
-from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream
-from webscout.AIbase import Provider
from webscout import exceptions
-from webscout.litagent import LitAgent
+from webscout.AIbase import Provider, Response
+from webscout.AIutel import AwesomePrompts, Optimizers, sanitize_stream # Import sanitize_stream
from webscout.conversation import Conversation
+from webscout.litagent import LitAgent
T = TypeVar('T')
@@ -29,12 +29,12 @@ def __init__(
is_conversation: bool = True,
max_tokens: int = 1280,
timeout: int = 30,
- intro: str = None,
- filepath: str = None,
+ intro: Optional[str] = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
proxies: dict = {},
history_offset: int = 10250,
- act: str = None,
+ act: Optional[str] = None,
model: str = "DeepSeek-R1-Distill-Qwen-32B",
temperature: float = 0.6,
top_p: float = 0.7,
@@ -57,7 +57,7 @@ def __init__(
)
# Initialize curl_cffi Session instead of cloudscraper
- self.session = Session()
+ self.session = Session()
self.is_conversation = is_conversation
self.max_tokens_to_sample = max_tokens
self.chat_endpoint = "https://api.yep.com/v1/chat/completions"
@@ -87,7 +87,7 @@ def __init__(
"Sec-CH-UA-Platform": f'"{self.fingerprint["platform"]}"',
"User-Agent": self.fingerprint["user_agent"],
}
-
+
# Create session cookies with unique identifiers
self.cookies = {"__Host-session": uuid.uuid4().hex, '__cf_bm': uuid.uuid4().hex}
@@ -114,13 +114,13 @@ def __init__(
def refresh_identity(self, browser: str = None):
"""
Refreshes the browser identity fingerprint.
-
+
Args:
browser: Specific browser to use for the new fingerprint
"""
browser = browser or self.fingerprint.get("browser_type", "chrome")
self.fingerprint = self.agent.generate_fingerprint(browser)
-
+
# Update headers with new fingerprint
self.headers.update({
"Accept": self.fingerprint["accept"],
@@ -129,13 +129,13 @@ def refresh_identity(self, browser: str = None):
"Sec-CH-UA-Platform": f'"{self.fingerprint["platform"]}"',
"User-Agent": self.fingerprint["user_agent"],
})
-
+
# Update session headers
self.session.headers.update(self.headers)
-
+
# Generate new cookies (will be passed in requests)
self.cookies = {"__Host-session": uuid.uuid4().hex, '__cf_bm': uuid.uuid4().hex}
-
+
return self.fingerprint
def ask(
@@ -143,9 +143,10 @@ def ask(
prompt: str,
stream: bool = False,
raw: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
- ) -> Union[Dict[str, Any], Generator]:
+ **kwargs: Any,
+ ) -> Response:
"""
Sends a prompt to the Yep API and returns the response.
@@ -237,7 +238,7 @@ def for_non_stream():
)
if raw:
return response.text
-
+
# Use sanitize_stream to parse the non-streaming JSON response
processed_stream = sanitize_stream(
data=response.text,
@@ -252,7 +253,7 @@ def for_non_stream():
if raw:
return content
content = content if isinstance(content, str) else ""
-
+
if content:
self.conversation.update_chat_history(prompt, content)
return {"text": content}
@@ -269,9 +270,10 @@ def chat(
self,
prompt: str,
stream: bool = False,
- optimizer: str = None,
+ optimizer: Optional[str] = None,
conversationally: bool = False,
raw: bool = False, # Added raw parameter
+ **kwargs: Any,
) -> Union[str, Generator[str, None, None]]:
"""
Initiates a chat with the Yep API using the provided prompt.
@@ -308,7 +310,7 @@ def for_non_stream():
return for_stream() if stream else for_non_stream()
- def get_message(self, response: dict) -> str:
+ def get_message(self, response: Response) -> str:
"""
Extracts the message content from the API response.
@@ -319,36 +321,19 @@ def get_message(self, response: dict) -> str:
Extracts and returns the message content from the response.
"""
if isinstance(response, dict):
- return response["text"]
+ return response.get("text", "")
elif isinstance(response, (str, bytes)):
- return response
+ return str(response)
else:
- raise TypeError(f"Unexpected response type: {type(response)}")
+ return str(response)
if __name__ == "__main__":
- # print("-" * 80)
- # print(f"{'Model':<50} {'Status':<10} {'Response'}")
- # print("-" * 80)
-
- # for model in YEPCHAT.AVAILABLE_MODELS:
- # try:
- # test_ai = YEPCHAT(model=model, timeout=60)
- # response = test_ai.chat("Say 'Hello' in one word")
- # response_text = response
-
- # if response_text and len(response_text.strip()) > 0:
- # status = "✓"
- # # Truncate response if too long
- # display_text = response_text.strip()[:50] + "..." if len(response_text.strip()) > 50 else response_text.strip()
- # else:
- # status = "✗"
- # display_text = "Empty or invalid response"
- # print(f"{model:<50} {status:<10} {display_text}")
- # except Exception as e:
- # print(f"{model:<50} {'✗':<10} {str(e)}")
ai = YEPCHAT(model="DeepSeek-R1-Distill-Qwen-32B", timeout=60)
response = ai.chat("Say 'Hello' in one word", raw=False, stream=True)
- for chunk in response:
+ if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)):
+ for chunk in response:
+ print(chunk, end='', flush=True)
+ else:
+ print(response)
- print(chunk, end='', flush=True)
\ No newline at end of file
diff --git a/webscout/__init__.py b/webscout/__init__.py
index 67cb7b54..d61e574a 100644
--- a/webscout/__init__.py
+++ b/webscout/__init__.py
@@ -1,30 +1,31 @@
# webscout/__init__.py
-from .search import *
-from .version import __version__
+from .AIauto import * # noqa: F403
+from .AIutel import * # noqa: F403
+from .client import Client
+from .Extra import * # noqa: F403
+from .litagent import LitAgent
+from .models import model
+from .optimizers import *
from .Provider import *
-from .AIauto import *
+from .Provider.AISEARCH import *
+from .Provider.STT import * # noqa: F403
from .Provider.TTI import *
from .Provider.TTS import *
-from .Provider.AISEARCH import *
-from .Provider.STT import *
-from .Extra import *
-from .optimizers import *
-from .swiftcli import *
-from .litagent import LitAgent
-from .client import Client
from .scout import *
+from .search import *
+from .swiftcli import *
+from .update_checker import check_for_updates
+from .version import __version__
from .zeroart import *
-from .AIutel import *
useragent = LitAgent()
-# Add update checker
-from .update_checker import check_for_updates
+
+
try:
update_message = check_for_updates()
if update_message:
print(update_message)
-except Exception as e:
+except Exception:
pass
-# Import models for easy access
-from .models import model
\ No newline at end of file
+
diff --git a/webscout/cli.py b/webscout/cli.py
index e53a4331..e4e707ca 100644
--- a/webscout/cli.py
+++ b/webscout/cli.py
@@ -1,25 +1,29 @@
-from .swiftcli import CLI, option, table_output, panel_output
+import sys
+from typing import Any, Dict, List, Optional, Type
+
+from rich import print as rprint
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+
from .search import (
- DuckDuckGoSearch,
- YepSearch,
- BingSearch,
- YahooSearch,
+ BaseSearch,
+ BingSearch,
Brave,
+ DuckDuckGoSearch,
Mojeek,
+ Wikipedia,
+ YahooSearch,
Yandex,
- Wikipedia
+ YepSearch,
)
+from .swiftcli import CLI, option
from .version import __version__
-from rich.console import Console
-from rich.table import Table
-from rich.panel import Panel
-from rich import print as rprint
-import sys
console = Console()
# Engine mapping
-ENGINES = {
+ENGINES: Dict[str, Type[BaseSearch]] = {
"ddg": DuckDuckGoSearch,
"duckduckgo": DuckDuckGoSearch,
"bing": BingSearch,
@@ -28,32 +32,32 @@
"mojeek": Mojeek,
"yandex": Yandex,
"wikipedia": Wikipedia,
- "yep": YepSearch
+ "yep": YepSearch,
}
-def _get_engine(name):
+def _get_engine(name: str) -> BaseSearch:
cls = ENGINES.get(name.lower())
if not cls:
rprint(f"[bold red]Error: Engine '{name}' not supported.[/bold red]")
rprint(f"Available engines: {', '.join(sorted(set(e for e in ENGINES.keys())))}")
sys.exit(1)
- return cls()
+ return cls() # type: ignore[arg-type]
-def _print_data(data, title="Search Results"):
+def _print_data(data: Any, title: str = "Search Results") -> None:
"""Prints data in a beautiful table."""
if not data:
rprint("[bold yellow]No results found.[/bold yellow]")
return
table = Table(title=title, show_header=True, header_style="bold magenta", show_lines=True)
-
+
if isinstance(data, list) and len(data) > 0:
if isinstance(data[0], dict):
keys = list(data[0].keys())
table.add_column("#", style="dim", width=4)
for key in keys:
table.add_column(key.capitalize())
-
+
for i, item in enumerate(data, 1):
row = [str(i)]
for key in keys:
@@ -73,44 +77,51 @@ def _print_data(data, title="Search Results"):
console.print(table)
-def _print_weather(data):
+def _print_weather(data: Dict[str, Any]) -> None:
"""Prints weather data in a clean panel."""
- current = data.get("current")
- if not current:
- rprint(f"[bold blue]Weather data:[/bold blue] {data}")
- return
-
+ # Be defensive when reading weather payloads
+ current = data.get("current") or {}
+ location = data.get("location", "Unknown")
+
+ temp = current.get('temperature_c', 'N/A')
+ feels_like = current.get('feels_like_c', 'N/A')
+ condition = current.get('condition', 'N/A')
+ humidity = current.get('humidity', 'N/A')
+ wind_speed = current.get('wind_speed_ms', 'N/A')
+ wind_dir = current.get('wind_direction', 'N/A')
+
weather_info = (
- f"[bold blue]Location:[/bold blue] {data['location']}\n"
- f"[bold blue]Temperature:[/bold blue] {current['temperature_c']}°C (Feels like {current['feels_like_c']}°C)\n"
- f"[bold blue]Condition:[/bold blue] {current['condition']}\n"
- f"[bold blue]Humidity:[/bold blue] {current['humidity']}%\n"
- f"[bold blue]Wind:[/bold blue] {current['wind_speed_ms']} m/s {current['wind_direction']}°"
+ f"[bold blue]Location:[/bold blue] {location}\n"
+ f"[bold blue]Temperature:[/bold blue] {temp}°C (Feels like {feels_like}°C)\n"
+ f"[bold blue]Condition:[/bold blue] {condition}\n"
+ f"[bold blue]Humidity:[/bold blue] {humidity}%\n"
+ f"[bold blue]Wind:[/bold blue] {wind_speed} m/s {wind_dir}°"
)
-
+
panel = Panel(weather_info, title="Current Weather", border_style="green")
console.print(panel)
-
- if "daily_forecast" in data:
+
+ if isinstance(data.get("daily_forecast"), list):
forecast_table = Table(title="5-Day Forecast", show_header=True, header_style="bold cyan")
forecast_table.add_column("Date")
forecast_table.add_column("Condition")
forecast_table.add_column("High")
forecast_table.add_column("Low")
-
- for day in data["daily_forecast"][:5]:
- forecast_table.add_row(
- day['date'],
- day['condition'],
- f"{day['max_temp_c']:.1f}°C",
- f"{day['min_temp_c']:.1f}°C"
- )
+
+ for day in data.get("daily_forecast", [])[:5]:
+ date = day.get('date', 'N/A')
+ condition = day.get('condition', 'N/A')
+ max_temp = day.get('max_temp_c')
+ min_temp = day.get('min_temp_c')
+ max_temp_str = f"{max_temp:.1f}°C" if isinstance(max_temp, (int, float)) else str(max_temp)
+ min_temp_str = f"{min_temp:.1f}°C" if isinstance(min_temp, (int, float)) else str(min_temp)
+ forecast_table.add_row(date, condition, max_temp_str, min_temp_str)
console.print(forecast_table)
-app = CLI(name="webscout", help="Search the web with a simple UI", version=__version__)
+app: CLI = CLI(name="webscout", help="Search the web with a simple UI", version=__version__)
@app.command()
-def version():
+def version() -> None:
"""Show the version of webscout."""
rprint(f"[bold cyan]webscout version:[/bold cyan] {__version__}")
@@ -121,14 +132,21 @@ def version():
@option("--safesearch", "-s", help="SafeSearch setting", default="moderate")
@option("--timelimit", "-t", help="Time limit for results", default=None)
@option("--max-results", "-m", help="Maximum number of results", type=int, default=10)
-def text(keywords: str, engine: str, region: str, safesearch: str, timelimit: str, max_results: int):
+def text(
+ keywords: str,
+ engine: str,
+ region: Optional[str] = None,
+ safesearch: str = "moderate",
+ timelimit: Optional[str] = None,
+ max_results: int = 10,
+) -> None:
"""Perform a text search."""
try:
search_engine = _get_engine(engine)
# Handle region defaults if not provided
if region is None:
region = "wt-wt" if engine.lower() in ["ddg", "duckduckgo"] else "us"
-
+
# Most engines use .text(), some use .search() or .run()
if hasattr(search_engine, 'text'):
results = search_engine.text(keywords, region=region, safesearch=safesearch, max_results=max_results)
@@ -136,7 +154,7 @@ def text(keywords: str, engine: str, region: str, safesearch: str, timelimit: st
results = search_engine.run(keywords, region=region, safesearch=safesearch, max_results=max_results)
else:
results = search_engine.search(keywords, max_results=max_results)
-
+
_print_data(results, title=f"{engine.upper()} Text Search: {keywords}")
except Exception as e:
rprint(f"[bold red]Error:[/bold red] {str(e)}")
@@ -145,7 +163,7 @@ def text(keywords: str, engine: str, region: str, safesearch: str, timelimit: st
@option("--keywords", "-k", help="Search keywords", required=True)
@option("--engine", "-e", help="Search engine (ddg, bing, yahoo)", default="ddg")
@option("--max-results", "-m", help="Maximum number of results", type=int, default=10)
-def images(keywords: str, engine: str, max_results: int):
+def images(keywords: str, engine: str, max_results: int) -> None:
"""Perform an images search."""
try:
search_engine = _get_engine(engine)
@@ -158,7 +176,7 @@ def images(keywords: str, engine: str, max_results: int):
@option("--keywords", "-k", help="Search keywords", required=True)
@option("--engine", "-e", help="Search engine (ddg, yahoo)", default="ddg")
@option("--max-results", "-m", help="Maximum number of results", type=int, default=10)
-def videos(keywords: str, engine: str, max_results: int):
+def videos(keywords: str, engine: str, max_results: int) -> None:
"""Perform a videos search."""
try:
search_engine = _get_engine(engine)
@@ -171,7 +189,7 @@ def videos(keywords: str, engine: str, max_results: int):
@option("--keywords", "-k", help="Search keywords", required=True)
@option("--engine", "-e", help="Search engine (ddg, bing, yahoo)", default="ddg")
@option("--max-results", "-m", help="Maximum number of results", type=int, default=10)
-def news(keywords: str, engine: str, max_results: int):
+def news(keywords: str, engine: str, max_results: int) -> None:
"""Perform a news search."""
try:
search_engine = _get_engine(engine)
@@ -183,7 +201,7 @@ def news(keywords: str, engine: str, max_results: int):
@app.command()
@option("--location", "-l", help="Location to get weather for", required=True)
@option("--engine", "-e", help="Search engine (ddg, yahoo)", default="ddg")
-def weather(location: str, engine: str):
+def weather(location: str, engine: str) -> None:
"""Get weather information."""
try:
search_engine = _get_engine(engine)
@@ -195,7 +213,7 @@ def weather(location: str, engine: str):
@app.command()
@option("--keywords", "-k", help="Search keywords", required=True)
@option("--engine", "-e", help="Search engine (ddg, yahoo)", default="ddg")
-def answers(keywords: str, engine: str):
+def answers(keywords: str, engine: str) -> None:
"""Perform an answers search."""
try:
search_engine = _get_engine(engine)
@@ -207,21 +225,16 @@ def answers(keywords: str, engine: str):
@app.command()
@option("--query", "-q", help="Search query", required=True)
@option("--engine", "-e", help="Search engine (ddg, bing, yahoo, yep)", default="ddg")
-def suggestions(query: str, engine: str):
+def suggestions(query: str, engine: str) -> None:
"""Get search suggestions."""
try:
search_engine = _get_engine(engine)
- # Some engines use 'keywords', some 'query'
- if engine.lower() in ["bing", "yep"]:
- results = search_engine.suggestions(query)
- else:
- results = search_engine.suggestions(query)
-
- # Format suggestions
+ results = search_engine.suggestions(query)
+
+ # Format suggestions (Bing-style dicts)
if isinstance(results, list) and results and isinstance(results[0], dict):
- # Bing format
results = [r.get("suggestion", str(r)) for r in results]
-
+
_print_data(results, title=f"{engine.upper()} Suggestions: {query}")
except Exception as e:
rprint(f"[bold red]Error:[/bold red] {str(e)}")
@@ -231,7 +244,7 @@ def suggestions(query: str, engine: str):
@option("--from", "-f", help="Language to translate from", default=None)
@option("--to", "-t", help="Language to translate to", default="en")
@option("--engine", "-e", help="Search engine (ddg, yahoo)", default="ddg")
-def translate(keywords: str, from_: str, to: str, engine: str):
+def translate(keywords: str, from_: Optional[str] = None, to: str = "en", engine: str = "ddg") -> None:
"""Perform translation."""
try:
search_engine = _get_engine(engine)
@@ -245,7 +258,7 @@ def translate(keywords: str, from_: str, to: str, engine: str):
@option("--place", "-p", help="Place name")
@option("--radius", "-r", help="Search radius (km)", type=int, default=0)
@option("--engine", "-e", help="Search engine (ddg, yahoo)", default="ddg")
-def maps(keywords: str, place: str, radius: int, engine: str):
+def maps(keywords: str, place: Optional[str] = None, radius: int = 0, engine: str = "ddg") -> None:
"""Perform a maps search."""
try:
search_engine = _get_engine(engine)
@@ -259,9 +272,10 @@ def maps(keywords: str, place: str, radius: int, engine: str):
@option("--keywords", "-k", help="Search keywords", required=True)
@option("--engine", "-e", help="Search engine", default="ddg")
@option("--max-results", "-m", help="Maximum results", type=int, default=10)
-def search(keywords: str, engine: str, max_results: int):
+def search(keywords: str, engine: str, max_results: int) -> None:
"""Unified search command across all engines."""
- text.run(keywords=keywords, engine=engine, max_results=max_results)
+ # Call the local `text` function implementation (not a command object)
+ text(keywords=keywords, engine=engine, max_results=max_results)
def main():
"""Main entry point for the CLI."""
@@ -272,4 +286,4 @@ def main():
sys.exit(1)
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/webscout/client.py b/webscout/client.py
index 2ffc7b8e..bdbbb7d5 100644
--- a/webscout/client.py
+++ b/webscout/client.py
@@ -12,29 +12,20 @@
- Full streaming support
"""
-import time
-import uuid
-import random
-import inspect
+import difflib
import importlib
+import inspect
import pkgutil
-import difflib
-from typing import List, Dict, Optional, Union, Generator, Any, Type, Tuple, Set
+import random
+from typing import Any, Dict, Generator, List, Optional, Set, Tuple, Type, Union
-from webscout.Provider.OPENAI import *
-from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseCompletions, BaseChat
+from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider
from webscout.Provider.OPENAI.utils import (
ChatCompletion,
ChatCompletionChunk,
- Choice,
- ChoiceDelta,
- ChatCompletionMessage,
- CompletionUsage,
)
-
-from webscout.Provider.TTI import *
-from webscout.Provider.TTI.base import TTICompatibleProvider, BaseImages
-from webscout.Provider.TTI.utils import ImageData, ImageResponse
+from webscout.Provider.TTI.base import BaseImages, TTICompatibleProvider
+from webscout.Provider.TTI.utils import ImageResponse
def load_openai_providers() -> Tuple[Dict[str, Type[OpenAICompatibleProvider]], Set[str]]:
@@ -359,7 +350,7 @@ def create(
first_chunk = next(response)
self._last_provider = resolved_provider.__name__
- def chained_gen(first, rest, pname):
+ def chained_gen(first, rest, pname) -> Any:
if self._client.print_provider_info:
print(f"\033[1;34m{pname}:{resolved_model}\033[0m\n")
yield first
@@ -718,7 +709,7 @@ def generate(
return response
except Exception:
continue
- raise RuntimeError(f"All image providers failed.")
+ raise RuntimeError("All image providers failed.")
def create(self, **kwargs) -> ImageResponse:
"""Alias for generate."""
diff --git a/webscout/conversation.py b/webscout/conversation.py
index 11c48b43..40e38a87 100644
--- a/webscout/conversation.py
+++ b/webscout/conversation.py
@@ -25,7 +25,7 @@ def __init__(
self,
status: bool = True,
max_tokens: int = 600,
- filepath: str = None,
+ filepath: Optional[str] = None,
update_file: bool = True,
):
"""Initializes Conversation
@@ -88,7 +88,7 @@ def __trim_chat_history(self, chat_history: str, intro: str) -> str:
else:
return chat_history
- def gen_complete_prompt(self, prompt: str, intro: str = None) -> str:
+ def gen_complete_prompt(self, prompt: str, intro: Optional[str] = None) -> str:
"""Generates a kinda like incomplete conversation
Args:
diff --git a/webscout/exceptions.py b/webscout/exceptions.py
index 5b823ae4..42b017f4 100644
--- a/webscout/exceptions.py
+++ b/webscout/exceptions.py
@@ -90,6 +90,18 @@ class FailedToGenerateResponseError(WebscoutE):
"""
pass
+class ProviderConnectionError(WebscoutE):
+ """
+ Exception raised when there are issues connecting to a specific provider.
+ """
+ pass
+
+class InvalidOptimizerError(WebscoutE):
+ """
+ Exception raised when an invalid or unavailable optimizer is requested.
+ """
+ pass
+
class InvalidAuthenticationError(Exception):
"""Custom exception for authentication errors (e.g., invalid API key, cookies)."""
pass
@@ -359,4 +371,4 @@ def __init__(self, video_id, requested_language_codes, transcript_data):
super().__init__(video_id, message.format(
requested_language_codes=requested_language_codes,
transcript_data=str(transcript_data)
- ))
\ No newline at end of file
+ ))
diff --git a/webscout/litagent/__init__.py b/webscout/litagent/__init__.py
index 43987e2d..ef16e095 100644
--- a/webscout/litagent/__init__.py
+++ b/webscout/litagent/__init__.py
@@ -4,7 +4,7 @@
Examples:
>>> from webscout import LitAgent
>>> agent = LitAgent()
->>>
+>>>
>>> # Get random user agents
>>> agent.random() # Random agent from any browser
>>> agent.mobile() # Random mobile device agent
@@ -22,8 +22,8 @@
"""
from .agent import LitAgent
-from .constants import BROWSERS, OS_VERSIONS, DEVICES, FINGERPRINTS
+from .constants import BROWSERS, DEVICES, FINGERPRINTS, OS_VERSIONS
agent = LitAgent()
-__all__ = ['LitAgent', 'agent', 'BROWSERS', 'OS_VERSIONS', 'DEVICES', 'FINGERPRINTS']
\ No newline at end of file
+__all__ = ['LitAgent', 'agent', 'BROWSERS', 'OS_VERSIONS', 'DEVICES', 'FINGERPRINTS']
diff --git a/webscout/litagent/agent.py b/webscout/litagent/agent.py
index 8dca6245..d75e024d 100644
--- a/webscout/litagent/agent.py
+++ b/webscout/litagent/agent.py
@@ -2,7 +2,7 @@
LitAgent: Advanced User Agent Generation and Management System.
This module provides a robust and flexible system for generating realistic,
-modern user agents and managing browser fingerprints for web scraping and
+modern user agents and managing browser fingerprints for web scraping and
automation purposes.
"""
@@ -10,7 +10,7 @@
import random
import threading
from datetime import datetime
-from typing import Any, Dict, List, Optional, Union, Tuple
+from typing import Any, Dict, List, Optional
from webscout.litagent.constants import BROWSERS, DEVICES, FINGERPRINTS, OS_VERSIONS
@@ -18,7 +18,7 @@
class LitAgent:
"""
A powerful and modern user agent generator for web scraping and automation.
-
+
LitAgent provides tools for generating randomized but realistic user agent strings,
managing proxy pools, rotating IPs, and simulating browser fingerprints to avoid
detection and rate limiting.
@@ -43,7 +43,7 @@ def __init__(self, thread_safe: bool = False):
"""
self.thread_safe = thread_safe
self.lock = threading.RLock() if thread_safe else None
-
+
# Internal state
self._blacklist: set = set()
self._whitelist: set = set()
@@ -54,7 +54,7 @@ def __init__(self, thread_safe: bool = False):
self._proxy_index: int = 0
self._history: List[str] = []
self._refresh_timer: Optional[threading.Timer] = None
-
+
# Usage statistics
self._stats = {
"total_generated": 100,
@@ -92,7 +92,7 @@ def _generate_agents(self, count: int) -> List[str]:
platform = f"X11; Linux {os_ver}"
agent = f"Mozilla/5.0 ({platform}) AppleWebKit/537.36 (KHTML, like Gecko) "
-
+
if browser == 'chrome':
agent += f"Chrome/{version}.0.0.0 Safari/537.36"
elif browser == 'firefox':
@@ -105,7 +105,7 @@ def _generate_agents(self, count: int) -> List[str]:
agent += f"Chrome/{version}.0.0.0 Safari/537.36 Brave/{version}.0.0.0"
elif browser == 'vivaldi':
agent += f"Chrome/{version}.0.0.0 Safari/537.36 Vivaldi/{version}.0.{random.randint(1000, 9999)}"
-
+
elif browser == 'safari':
device = random.choice(['mac', 'ios'])
if device == 'mac':
@@ -115,7 +115,7 @@ def _generate_agents(self, count: int) -> List[str]:
ver = random.choice(OS_VERSIONS.get('ios', ["17_0"]))
device_name = random.choice(['iPhone', 'iPad'])
agent = f"Mozilla/5.0 ({device_name}; CPU OS {ver} like Mac OS X) "
-
+
agent += f"AppleWebKit/{version}.1.15 (KHTML, like Gecko) Version/{version//10}.0 Safari/{version}.1.15"
agents.append(agent)
@@ -163,7 +163,7 @@ def random(self) -> str:
if not pool:
# Fallback if somehow empty
pool = self._generate_agents(1)
-
+
agent = random.choice(pool)
self._update_stats()
self._add_to_history(agent)
@@ -189,7 +189,7 @@ def browser(self, name: str) -> str:
matched = self.custom(browser=name)
else:
matched = random.choice(matching_agents)
-
+
self._update_stats(browser_type=name)
self._add_to_history(matched)
return matched
@@ -248,7 +248,7 @@ def custom(self, browser: Optional[str] = None, version: Optional[str] = None,
browser = browser.lower() if browser else 'chrome'
v_range = BROWSERS.get(browser, (100, 130))
v_num = int(version.split('.')[0]) if version else random.randint(*v_range)
-
+
os = os.lower() if os else random.choice(['windows', 'mac', 'linux'])
os_ver = os_version or random.choice(OS_VERSIONS.get(os, ["10.0"]))
device_type = (device_type or 'desktop').lower()
@@ -268,7 +268,7 @@ def custom(self, browser: Optional[str] = None, version: Optional[str] = None,
platform = "Windows NT 10.0; Win64; x64"
agent = f"Mozilla/5.0 ({platform}) AppleWebKit/537.36 (KHTML, like Gecko) "
-
+
if browser == 'chrome':
agent += f"Chrome/{v_num}.0.0.0 Safari/537.36"
elif browser == 'firefox':
@@ -295,7 +295,7 @@ def generate_fingerprint(self, browser: Optional[str] = None) -> Dict[str, str]:
"""
ua = self.browser(browser) if browser else self.random()
ip = self.rotate_ip()
-
+
sec_ch_ua = ""
for b_name in FINGERPRINTS.get("sec_ch_ua", {}):
if b_name in ua.lower():
@@ -325,7 +325,7 @@ def smart_tv(self) -> str:
agent = f"Mozilla/5.0 (Web0S; {tv}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
else:
agent = f"Mozilla/5.0 (Linux; {tv}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36"
-
+
self._update_stats(device_type="tv")
return agent
@@ -338,7 +338,7 @@ def gaming(self) -> str:
agent = f"Mozilla/5.0 (Windows NT 10.0; Win64; x64; Xbox; {console}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edge/110.0.1587.41"
else:
agent = self.random()
-
+
self._update_stats(device_type="console")
return agent
@@ -349,7 +349,7 @@ def wearable(self) -> str:
agent = "Mozilla/5.0 (AppleWatch; CPU WatchOS like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/10.0 Mobile/15E148 Safari/605.1"
else:
agent = f"Mozilla/5.0 (Linux; {dev}) AppleWebKit/537.36 (KHTML, like Gecko)"
-
+
self._update_stats(device_type="wearable")
return agent
@@ -406,7 +406,7 @@ def rotate_proxy(self) -> Optional[str]:
"""Rotate through and return the next proxy from the pool."""
if not self._proxy_pool:
return None
-
+
def rot():
proxy = self._proxy_pool[self._proxy_index]
self._proxy_index = (self._proxy_index + 1) % len(self._proxy_pool)
diff --git a/webscout/litagent/constants.py b/webscout/litagent/constants.py
index 5f8bef22..6d6c53f3 100644
--- a/webscout/litagent/constants.py
+++ b/webscout/litagent/constants.py
@@ -37,10 +37,10 @@
# Browser fingerprinting components
FINGERPRINTS = {
"accept_language": [
- "en-US,en;q=0.9",
- "en-GB,en;q=0.8,en-US;q=0.6",
+ "en-US,en;q=0.9",
+ "en-GB,en;q=0.8,en-US;q=0.6",
"es-ES,es;q=0.9,en;q=0.8",
- "fr-FR,fr;q=0.9,en;q=0.8",
+ "fr-FR,fr;q=0.9,en;q=0.8",
"de-DE,de;q=0.9,en;q=0.8"
],
"accept": [
@@ -57,4 +57,4 @@
"platforms": [
"Windows", "macOS", "Linux", "Android", "iOS"
]
-}
\ No newline at end of file
+}
diff --git a/webscout/models.py b/webscout/models.py
index 966eb4bd..d9da9a25 100644
--- a/webscout/models.py
+++ b/webscout/models.py
@@ -1,6 +1,7 @@
import importlib
import pkgutil
-from typing import Dict, List, Any, Union
+from typing import Any, Dict, List, Union
+
from webscout.AIbase import Provider, TTSProvider
# Import TTI base class
@@ -15,41 +16,41 @@ class _LLMModels:
"""
A class for managing LLM provider models in the webscout package.
"""
-
+
def list(self) -> Dict[str, List[str]]:
"""
Gets all available models from each provider that has an AVAILABLE_MODELS attribute.
-
+
Returns:
Dictionary mapping provider names to their available models
"""
return self._get_provider_models()
-
+
def get(self, provider_name: str) -> List[str]:
"""
Gets all available models for a specific provider.
-
+
Args:
provider_name: The name of the provider
-
+
Returns:
List of available models for the provider
"""
all_models = self._get_provider_models()
return all_models.get(provider_name, [])
-
+
def summary(self) -> Dict[str, int]:
"""
Returns a summary of available providers and models.
-
+
Returns:
Dictionary with provider and model counts
"""
provider_models = self._get_provider_models()
total_providers = len(provider_models)
- total_models = sum(len(models) if isinstance(models, (list, tuple, set))
+ total_models = sum(len(models) if isinstance(models, (list, tuple, set))
else 1 for models in provider_models.values())
-
+
return {
"providers": total_providers,
"models": total_models,
@@ -58,39 +59,39 @@ def summary(self) -> Dict[str, int]:
for provider, models in provider_models.items()
}
}
-
+
def providers(self) -> Dict[str, Dict[str, Any]]:
"""
Gets detailed information about all LLM providers including models, parameters, and metadata.
-
+
Returns:
Dictionary mapping provider names to detailed provider information
"""
return self._get_provider_details()
-
+
def provider(self, provider_name: str) -> Dict[str, Any]:
"""
Gets detailed information about a specific LLM provider.
-
+
Args:
provider_name: The name of the provider
-
+
Returns:
Dictionary with detailed provider information
"""
all_providers = self._get_provider_details()
return all_providers.get(provider_name, {})
-
+
def _get_provider_models(self) -> Dict[str, List[str]]:
"""
Internal method to get all available models from each provider.
-
+
Returns:
Dictionary mapping provider names to their available models
"""
provider_models = {}
provider_package = importlib.import_module("webscout.Provider")
-
+
for _, module_name, _ in pkgutil.iter_modules(provider_package.__path__):
try:
module = importlib.import_module(f"webscout.Provider.{module_name}")
@@ -115,19 +116,19 @@ def _get_provider_models(self) -> Dict[str, List[str]]:
provider_models[attr_name] = []
except Exception:
pass
-
+
return provider_models
-
+
def _get_provider_details(self) -> Dict[str, Dict[str, Any]]:
"""
Internal method to get detailed information about all LLM providers.
-
+
Returns:
Dictionary mapping provider names to detailed provider information
"""
provider_details = {}
provider_package = importlib.import_module("webscout.Provider")
-
+
for _, module_name, _ in pkgutil.iter_modules(provider_package.__path__):
try:
module = importlib.import_module(f"webscout.Provider.{module_name}")
@@ -155,21 +156,21 @@ def _get_provider_details(self) -> Dict[str, Dict[str, Any]]:
models = list(available_models)
else:
models = [str(available_models)]
-
+
# Sort models
models = sorted(models)
-
+
# Get supported parameters (common OpenAI-compatible parameters)
supported_params = [
- "model", "messages", "max_tokens", "temperature", "top_p",
+ "model", "messages", "max_tokens", "temperature", "top_p",
"presence_penalty", "frequency_penalty", "stop", "stream", "user"
]
-
+
# Get additional metadata
metadata = {}
if hasattr(attr, '__doc__') and attr.__doc__:
metadata['description'] = attr.__doc__.strip().split('\n')[0]
-
+
provider_details[attr_name] = {
"name": attr_name,
"class": attr.__name__,
@@ -181,50 +182,50 @@ def _get_provider_details(self) -> Dict[str, Dict[str, Any]]:
}
except Exception:
pass
-
+
return provider_details
class _TTSModels:
"""
A class for managing TTS provider voices in the webscout package.
"""
-
+
def list(self) -> Dict[str, List[str]]:
"""
Gets all available voices from each TTS provider that has an all_voices attribute.
-
+
Returns:
Dictionary mapping TTS provider names to their available voices
"""
return self._get_tts_voices()
-
+
def get(self, provider_name: str) -> Union[List[str], Dict[str, str]]:
"""
Gets all available voices for a specific TTS provider.
-
+
Args:
provider_name: The name of the TTS provider
-
+
Returns:
List or Dictionary of available voices for the provider
"""
all_voices = self._get_tts_voices()
return all_voices.get(provider_name, [])
-
+
def summary(self) -> Dict[str, Any]:
"""
Returns a summary of available TTS providers and voices.
-
+
Returns:
Dictionary with provider and voice counts
"""
provider_voices = self._get_tts_voices()
total_providers = len(provider_voices)
-
+
# Count voices, handling both list and dict formats
total_voices = 0
provider_voice_counts = {}
-
+
for provider, voices in provider_voices.items():
if isinstance(voices, dict):
count = len(voices)
@@ -232,29 +233,29 @@ def summary(self) -> Dict[str, Any]:
count = len(voices)
else:
count = 1
-
+
total_voices += count
provider_voice_counts[provider] = count
-
+
return {
"providers": total_providers,
"voices": total_voices,
"provider_voice_counts": provider_voice_counts
}
-
+
def _get_tts_voices(self) -> Dict[str, Union[List[str], Dict[str, str]]]:
"""
Internal method to get all available voices from each TTS provider.
-
+
Returns:
Dictionary mapping TTS provider names to their available voices
"""
provider_voices = {}
-
+
try:
# Import the TTS package specifically
tts_package = importlib.import_module("webscout.Provider.TTS")
-
+
# Iterate through TTS modules
for _, module_name, _ in pkgutil.iter_modules(tts_package.__path__):
try:
@@ -266,74 +267,74 @@ def _get_tts_voices(self) -> Dict[str, Union[List[str], Dict[str, str]]]:
if hasattr(attr, 'all_voices'):
voices = attr.all_voices
provider_voices[attr_name] = voices
- except Exception as e:
+ except Exception:
pass
- except Exception as e:
+ except Exception:
pass
-
+
return provider_voices
class _TTIModels:
"""
A class for managing TTI (Text-to-Image) provider models in the webscout package.
"""
-
+
def list(self) -> Dict[str, List[str]]:
"""
Gets all available models from each TTI provider that has an AVAILABLE_MODELS attribute.
-
+
Returns:
Dictionary mapping TTI provider names to their available models
"""
return self._get_tti_models()
-
+
def get(self, provider_name: str) -> List[str]:
"""
Gets all available models for a specific TTI provider.
-
+
Args:
provider_name: The name of the TTI provider
-
+
Returns:
List of available models for the provider
"""
all_models = self._get_tti_models()
return all_models.get(provider_name, [])
-
+
def providers(self) -> Dict[str, Dict[str, Any]]:
"""
Gets detailed information about all TTI providers including models, parameters, and metadata.
-
+
Returns:
Dictionary mapping provider names to detailed provider information
"""
return self._get_tti_provider_details()
-
+
def provider(self, provider_name: str) -> Dict[str, Any]:
"""
Gets detailed information about a specific TTI provider.
-
+
Args:
provider_name: The name of the TTI provider
-
+
Returns:
Dictionary with detailed provider information
"""
all_providers = self._get_tti_provider_details()
return all_providers.get(provider_name, {})
-
+
def summary(self) -> Dict[str, int]:
"""
Returns a summary of available TTI providers and models.
-
+
Returns:
Dictionary with provider and model counts
"""
provider_models = self._get_tti_models()
total_providers = len(provider_models)
- total_models = sum(len(models) if isinstance(models, (list, tuple, set))
+ total_models = sum(len(models) if isinstance(models, (list, tuple, set))
else 1 for models in provider_models.values())
-
+
return {
"providers": total_providers,
"models": total_models,
@@ -342,26 +343,26 @@ def summary(self) -> Dict[str, int]:
for provider, models in provider_models.items()
}
}
-
+
def _get_tti_models(self) -> Dict[str, List[str]]:
"""
Internal method to get all available models from each TTI provider.
-
+
Returns:
Dictionary mapping TTI provider names to their available models
"""
if not TTI_AVAILABLE:
return {}
-
+
provider_models = {}
tti_package = importlib.import_module("webscout.Provider.TTI")
-
+
for _, module_name, _ in pkgutil.iter_modules(tti_package.__path__):
try:
module = importlib.import_module(f"webscout.Provider.TTI.{module_name}")
for attr_name in dir(module):
attr = getattr(module, attr_name)
- if (isinstance(attr, type) and BaseImages and
+ if (isinstance(attr, type) and BaseImages and
issubclass(attr, BaseImages) and attr != BaseImages):
if hasattr(attr, 'AVAILABLE_MODELS'):
# Convert any sets to lists to ensure serializability
@@ -371,28 +372,28 @@ def _get_tti_models(self) -> Dict[str, List[str]]:
provider_models[attr_name] = models
except Exception:
pass
-
+
return provider_models
-
+
def _get_tti_provider_details(self) -> Dict[str, Dict[str, Any]]:
"""
Internal method to get detailed information about all TTI providers.
-
+
Returns:
Dictionary mapping provider names to detailed provider information
"""
if not TTI_AVAILABLE:
return {}
-
+
provider_details = {}
tti_package = importlib.import_module("webscout.Provider.TTI")
-
+
for _, module_name, _ in pkgutil.iter_modules(tti_package.__path__):
try:
module = importlib.import_module(f"webscout.Provider.TTI.{module_name}")
for attr_name in dir(module):
attr = getattr(module, attr_name)
- if (isinstance(attr, type) and BaseImages and
+ if (isinstance(attr, type) and BaseImages and
issubclass(attr, BaseImages) and attr != BaseImages):
# Get available models
models = []
@@ -415,21 +416,21 @@ def _get_tti_provider_details(self) -> Dict[str, Dict[str, Any]]:
models = list(available_models)
else:
models = [str(available_models)]
-
+
# Sort models
models = sorted(models)
-
+
# Get supported parameters (common TTI parameters)
supported_params = [
- "prompt", "model", "n", "size", "response_format", "user",
+ "prompt", "model", "n", "size", "response_format", "user",
"style", "aspect_ratio", "timeout", "image_format", "seed"
]
-
+
# Get additional metadata
metadata = {}
if hasattr(attr, '__doc__') and attr.__doc__:
metadata['description'] = attr.__doc__.strip().split('\n')[0]
-
+
provider_details[attr_name] = {
"name": attr_name,
"class": attr.__name__,
@@ -441,7 +442,7 @@ def _get_tti_provider_details(self) -> Dict[str, Dict[str, Any]]:
}
except Exception:
pass
-
+
return provider_details
# Create singleton instances
@@ -451,10 +452,10 @@ def _get_tti_provider_details(self) -> Dict[str, Dict[str, Any]]:
# Container class for all model types
class Models:
- def __init__(self):
- self.llm = llm
- self.tts = tts
- self.tti = tti
+ def __init__(self) -> None:
+ self.llm: _LLMModels = llm
+ self.tts: _TTSModels = tts
+ self.tti: _TTIModels = tti
# Create a singleton instance
model = Models()
diff --git a/webscout/optimizers.py b/webscout/optimizers.py
index 5cf8e124..3fbaec1e 100644
--- a/webscout/optimizers.py
+++ b/webscout/optimizers.py
@@ -71,4 +71,4 @@ def coder(prompt: str) -> str:
Output:
"""
- )
\ No newline at end of file
+ )
diff --git a/webscout/prompt_manager.py b/webscout/prompt_manager.py
index 3e8a4538..28072d54 100644
--- a/webscout/prompt_manager.py
+++ b/webscout/prompt_manager.py
@@ -3,11 +3,10 @@
import json
import threading
-from pathlib import Path
-from typing import Optional, Dict, Union
-from functools import lru_cache
from datetime import datetime, timedelta
-
+from functools import lru_cache
+from pathlib import Path
+from typing import Dict, Optional, Union
try:
from curl_cffi.requests import Session
@@ -52,7 +51,7 @@ def __init__(
self._cache_lock = threading.RLock()
self._file_lock = threading.Lock()
self._max_workers = max_workers
-
+
self._max_workers = max_workers
if CURL_AVAILABLE:
self.session = Session(timeout=timeout, impersonate=impersonate)
@@ -95,7 +94,7 @@ def _load_prompts(self) -> Dict[Union[str, int], str]:
with self._cache_lock:
if self._cache:
return self._cache.copy()
-
+
# Fallback to file if cache is empty
self._load_cache()
with self._cache_lock:
@@ -117,45 +116,45 @@ def update_prompts_from_online(self, force: bool = False) -> bool:
return True
console.print("[cyan]Updating prompts...[/cyan]")
-
+
# Fetch new prompts with timeout
response = self.session.get(self.repo_url, timeout=self.timeout)
response.raise_for_status()
-
+
new_prompts = response.json()
if not isinstance(new_prompts, dict):
raise ValueError("Invalid response format")
-
+
# Efficient merge with existing prompts
existing_prompts = self._load_prompts()
-
+
# Build optimized structure
merged_prompts = {}
string_keys = []
-
+
# Add existing string keys
for key, value in existing_prompts.items():
if isinstance(key, str):
merged_prompts[key] = value
string_keys.append(key)
-
+
# Merge new prompts (prioritize new over existing)
for key, value in new_prompts.items():
if isinstance(key, str):
merged_prompts[key] = value
if key not in string_keys:
string_keys.append(key)
-
+
# Add numeric indices for fast access
for i, key in enumerate(string_keys):
merged_prompts[i] = merged_prompts[key]
-
+
self._save_prompts(merged_prompts)
self._last_update = datetime.now()
-
+
console.print(f"[green]Updated {len([k for k in merged_prompts if isinstance(k, str)])} prompts successfully![/green]")
return True
-
+
except Exception as e:
error_msg = str(e)
if hasattr(e, 'response') and e.response is not None:
@@ -171,7 +170,7 @@ def get_act(
use_cache: bool = True
) -> Optional[str]:
"""Get prompt with LRU caching for performance.
-
+
Args:
key: Prompt name or index
default: Default value if not found
@@ -181,7 +180,7 @@ def get_act(
if use_cache:
return self._get_cached(key, default, case_insensitive)
return self._get_uncached(key, default, case_insensitive)
-
+
def _get_uncached(
self,
key: Union[str, int],
@@ -191,23 +190,23 @@ def _get_uncached(
"""Core get logic without caching."""
with self._cache_lock:
prompts = self._cache if self._cache else self._load_prompts()
-
+
# Fast direct lookup
if key in prompts:
return prompts[key]
-
+
# Case-insensitive search for string keys
if isinstance(key, str) and case_insensitive:
key_lower = key.lower()
for k, v in prompts.items():
if isinstance(k, str) and k.lower() == key_lower:
return v
-
+
return default
def add_prompt(self, name: str, prompt: str, validate: bool = True) -> bool:
"""Add a new prompt with validation and deduplication.
-
+
Args:
name: Name of the prompt
prompt: The prompt text
@@ -217,30 +216,30 @@ def add_prompt(self, name: str, prompt: str, validate: bool = True) -> bool:
if not name or not prompt:
console.print("[red]Name and prompt cannot be empty![/red]")
return False
-
+
if len(name) > 100 or len(prompt) > 10000:
console.print("[red]Name too long (max 100) or prompt too long (max 10000)[/red]")
return False
-
+
with self._cache_lock:
prompts = self._load_prompts()
-
+
# Check for existing prompt with same content
if validate:
for existing_name, existing_prompt in prompts.items():
if isinstance(existing_name, str) and existing_prompt == prompt:
console.print(f"[yellow]Prompt with same content exists: '{existing_name}'[/yellow]")
return False
-
+
prompts[name] = prompt
-
+
# Update numeric indices
string_keys = [k for k in prompts.keys() if isinstance(k, str)]
for i, key in enumerate(string_keys):
prompts[i] = prompts[key]
-
+
self._save_prompts(prompts)
-
+
console.print(f"[green]Added prompt: '{name}'[/green]")
return True
@@ -251,7 +250,7 @@ def delete_prompt(
raise_not_found: bool = False
) -> bool:
"""Delete a prompt with proper cleanup.
-
+
Args:
name: Name or index of prompt to delete
case_insensitive: Enable case-insensitive matching
@@ -259,33 +258,33 @@ def delete_prompt(
"""
with self._cache_lock:
prompts = self._load_prompts()
-
+
# Handle direct key match
if name in prompts:
del prompts[name]
-
+
# Rebuild numeric indices after deletion
string_keys = [k for k in prompts.keys() if isinstance(k, str)]
# Remove old numeric indices
numeric_keys = [k for k in prompts.keys() if isinstance(k, int)]
for key in numeric_keys:
del prompts[key]
-
+
# Add fresh numeric indices
for i, key in enumerate(string_keys):
prompts[i] = prompts[key]
-
+
self._save_prompts(prompts)
console.print(f"[green]Deleted prompt: '{name}'[/green]")
return True
-
+
# Handle case-insensitive match
if isinstance(name, str) and case_insensitive:
name_lower = name.lower()
for k in list(prompts.keys()):
if isinstance(k, str) and k.lower() == name_lower:
return self.delete_prompt(k, case_insensitive=False, raise_not_found=raise_not_found)
-
+
if raise_not_found:
raise KeyError(f"Prompt '{name}' not found!")
console.print(f"[yellow]Prompt '{name}' not found![/yellow]")
@@ -297,46 +296,46 @@ def all_acts(self) -> Dict[Union[str, int], str]:
with self._cache_lock:
if self._cache:
return self._cache.copy()
-
+
prompts = self._load_prompts()
if not prompts:
self.update_prompts_from_online()
prompts = self._load_prompts()
-
+
return prompts.copy()
def show_acts(self, search: Optional[str] = None, limit: int = 100) -> None:
"""Display prompts with optimized filtering and pagination.
-
+
Args:
search: Filter by search term
limit: Maximum number of prompts to display
"""
prompts = self.all_acts
-
+
# Build filtered list efficiently
filtered_items = []
search_lower = search.lower() if search else None
-
+
for key, value in prompts.items():
if isinstance(key, int):
continue
-
+
if search_lower:
- if (search_lower not in key.lower() and
+ if (search_lower not in key.lower() and
search_lower not in value.lower()):
continue
-
+
preview = value[:80] + "..." if len(value) > 80 else value
filtered_items.append((str(key), preview))
-
+
if len(filtered_items) >= limit:
break
-
+
if not filtered_items:
console.print("[yellow]No prompts found[/yellow]")
return
-
+
table = Table(
title=f"Awesome Prompts ({len(filtered_items)} shown)",
show_header=True,
@@ -344,10 +343,10 @@ def show_acts(self, search: Optional[str] = None, limit: int = 100) -> None:
)
table.add_column("Name", style="green", max_width=30)
table.add_column("Preview", style="yellow", max_width=50)
-
+
for name, preview in filtered_items:
table.add_row(name, preview)
-
+
console.print(table)
def get_random_act(self) -> Optional[str]:
@@ -359,4 +358,4 @@ def get_random_act(self) -> Optional[str]:
import random
return prompts[random.choice(string_keys)]
- # End of class AwesomePrompts
\ No newline at end of file
+ # End of class AwesomePrompts
diff --git a/webscout/search/__init__.py b/webscout/search/__init__.py
index 6e8c9f3e..f0d5b2d8 100644
--- a/webscout/search/__init__.py
+++ b/webscout/search/__init__.py
@@ -1,44 +1,43 @@
"""Webscout search module - unified search interfaces."""
from .base import BaseSearch, BaseSearchEngine
-from .duckduckgo_main import DuckDuckGoSearch
-from .yep_main import YepSearch
from .bing_main import BingSearch
-from .yahoo_main import YahooSearch
+from .duckduckgo_main import DuckDuckGoSearch
# Import new search engines
from .engines.brave import Brave
from .engines.mojeek import Mojeek
-
-from .engines.yandex import Yandex
from .engines.wikipedia import Wikipedia
+from .engines.yandex import Yandex
# Import result models
from .results import (
- TextResult,
+ BooksResult,
ImagesResult,
- VideosResult,
NewsResult,
- BooksResult,
+ TextResult,
+ VideosResult,
)
+from .yahoo_main import YahooSearch
+from .yep_main import YepSearch
__all__ = [
# Base classes
"BaseSearch",
"BaseSearchEngine",
-
+
# Main search interfaces
"DuckDuckGoSearch",
"YepSearch",
"BingSearch",
"YahooSearch",
-
+
# Individual engines
"Brave",
"Mojeek",
"Yandex",
"Wikipedia",
-
+
# Result models
"TextResult",
"ImagesResult",
diff --git a/webscout/search/base.py b/webscout/search/base.py
index d1e2e5b7..6d663551 100644
--- a/webscout/search/base.py
+++ b/webscout/search/base.py
@@ -2,12 +2,13 @@
from __future__ import annotations
-from litprinter import ic
from abc import ABC, abstractmethod
from collections.abc import Mapping
from functools import cached_property
from typing import Any, Generic, Literal, TypeVar
+from litprinter import ic
+
try:
from lxml import html
from lxml.etree import HTMLParser as LHTMLParser
@@ -41,7 +42,7 @@ class BaseSearchEngine(ABC, Generic[T]):
def __init__(self, proxy: str | None = None, timeout: int | None = None, verify: bool = True):
"""Initialize search engine.
-
+
Args:
proxy: Proxy URL (supports http/https/socks5).
timeout: Request timeout in seconds.
@@ -75,14 +76,16 @@ def request(self, method: str, url: str, **kwargs: Any) -> str | None:
response = self.http_client.request(method, url, **kwargs) # type: ignore
return response.text
except Exception as ex:
- ic.configureOutput(prefix='ERROR| '); ic(f"Error in {self.name} request: {ex}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Error in {self.name} request: {ex}")
return None
@cached_property
def parser(self) -> Any:
"""Get HTML parser."""
if not LXML_AVAILABLE:
- ic.configureOutput(prefix='WARNING| '); ic("lxml not available, HTML parsing disabled")
+ ic.configureOutput(prefix='WARNING| ')
+ ic("lxml not available, HTML parsing disabled")
return None
return LHTMLParser(remove_blank_text=True, remove_comments=True, remove_pis=True, collect_ids=False)
@@ -100,13 +103,13 @@ def extract_results(self, html_text: str) -> list[T]:
"""Extract search results from html text."""
if not LXML_AVAILABLE:
raise ImportError("lxml is required for result extraction")
-
+
html_text = self.pre_process_html(html_text)
tree = self.extract_tree(html_text)
-
+
results = []
items = tree.xpath(self.items_xpath) if self.items_xpath else []
-
+
for item in items:
result = self.result_type()
for key, xpath in self.elements_xpath.items():
@@ -117,9 +120,10 @@ def extract_results(self, html_text: str) -> list[T]:
value = "".join(data) if isinstance(data, list) else data
setattr(result, key, value.strip() if isinstance(value, str) else value)
except Exception as ex:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Error extracting {key}: {ex}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Error extracting {key}: {ex}")
results.append(result)
-
+
return results
def post_extract_results(self, results: list[T]) -> list[T]:
@@ -154,41 +158,41 @@ class BaseSearch(ABC):
"""Base class for synchronous search engines (legacy)."""
@abstractmethod
- def text(self, *args, **kwargs) -> list[dict[str, str]]:
+ def text(self, *args, **kwargs) -> list[Any]:
"""Text search."""
raise NotImplementedError
@abstractmethod
- def images(self, *args, **kwargs) -> list[dict[str, str]]:
+ def images(self, *args, **kwargs) -> list[Any]:
"""Images search."""
raise NotImplementedError
@abstractmethod
- def videos(self, *args, **kwargs) -> list[dict[str, str]]:
+ def videos(self, *args, **kwargs) -> list[Any]:
"""Videos search."""
raise NotImplementedError
@abstractmethod
- def news(self, *args, **kwargs) -> list[dict[str, str]]:
+ def news(self, *args, **kwargs) -> list[Any]:
"""News search."""
raise NotImplementedError
@abstractmethod
- def answers(self, *args, **kwargs) -> list[dict[str, str]]:
+ def answers(self, *args, **kwargs) -> list[Any]:
"""Instant answers."""
raise NotImplementedError
@abstractmethod
- def suggestions(self, *args, **kwargs) -> list[dict[str, str]]:
+ def suggestions(self, *args, **kwargs) -> list[Any]:
"""Suggestions."""
raise NotImplementedError
@abstractmethod
- def maps(self, *args, **kwargs) -> list[dict[str, str]]:
+ def maps(self, *args, **kwargs) -> list[Any]:
"""Maps search."""
raise NotImplementedError
@abstractmethod
- def translate(self, *args, **kwargs) -> list[dict[str, str]]:
+ def translate(self, *args, **kwargs) -> list[Any]:
"""Translate."""
- raise NotImplementedError
\ No newline at end of file
+ raise NotImplementedError
diff --git a/webscout/search/bing_main.py b/webscout/search/bing_main.py
index ac94e90b..01df4df0 100644
--- a/webscout/search/bing_main.py
+++ b/webscout/search/bing_main.py
@@ -1,26 +1,29 @@
"""Bing unified search interface."""
from __future__ import annotations
+
from typing import Dict, List, Optional
+
from .base import BaseSearch
-from .engines.bing.text import BingTextSearch
from .engines.bing.images import BingImagesSearch
from .engines.bing.news import BingNewsSearch
from .engines.bing.suggestions import BingSuggestionsSearch
+from .engines.bing.text import BingTextSearch
+from .results import ImagesResult, NewsResult, TextResult
class BingSearch(BaseSearch):
"""Unified Bing search interface."""
- def text(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None, unique: bool = True) -> List[Dict[str, str]]:
+ def text(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None, unique: bool = True) -> List[TextResult]:
search = BingTextSearch()
return search.run(keywords, region, safesearch, max_results, unique=unique)
- def images(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+ def images(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[ImagesResult]:
search = BingImagesSearch()
return search.run(keywords, region, safesearch, max_results)
- def news(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+ def news(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[NewsResult]:
search = BingNewsSearch()
return search.run(keywords, region, safesearch, max_results)
@@ -39,4 +42,4 @@ def translate(self, keywords: str, from_lang: Optional[str] = None, to_lang: str
raise NotImplementedError("Translate not implemented for Bing")
def videos(self, *args, **kwargs) -> List[Dict[str, str]]:
- raise NotImplementedError("Videos not implemented for Bing")
\ No newline at end of file
+ raise NotImplementedError("Videos not implemented for Bing")
diff --git a/webscout/search/duckduckgo_main.py b/webscout/search/duckduckgo_main.py
index 0559fd84..63a7a326 100644
--- a/webscout/search/duckduckgo_main.py
+++ b/webscout/search/duckduckgo_main.py
@@ -1,35 +1,38 @@
"""DuckDuckGo unified search interface."""
from __future__ import annotations
-from typing import Dict, List, Optional
+
+from typing import Any, Dict, List, Optional, Union
+
from .base import BaseSearch
-from .engines.duckduckgo.text import DuckDuckGoTextSearch
+from .engines.duckduckgo.answers import DuckDuckGoAnswers
from .engines.duckduckgo.images import DuckDuckGoImages
-from .engines.duckduckgo.videos import DuckDuckGoVideos
+from .engines.duckduckgo.maps import DuckDuckGoMaps
from .engines.duckduckgo.news import DuckDuckGoNews
-from .engines.duckduckgo.answers import DuckDuckGoAnswers
from .engines.duckduckgo.suggestions import DuckDuckGoSuggestions
-from .engines.duckduckgo.maps import DuckDuckGoMaps
+from .engines.duckduckgo.text import DuckDuckGoTextSearch
from .engines.duckduckgo.translate import DuckDuckGoTranslate
+from .engines.duckduckgo.videos import DuckDuckGoVideos
from .engines.duckduckgo.weather import DuckDuckGoWeather
+from .results import ImagesResult, NewsResult, TextResult, VideosResult
class DuckDuckGoSearch(BaseSearch):
"""Unified DuckDuckGo search interface."""
- def text(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, backend: str = "api", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+ def text(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, backend: str = "api", max_results: Optional[int] = None) -> List[TextResult]:
search = DuckDuckGoTextSearch()
return search.run(keywords, region, safesearch, timelimit, backend, max_results)
- def images(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, size: Optional[str] = None, color: Optional[str] = None, type_image: Optional[str] = None, layout: Optional[str] = None, license_image: Optional[str] = None, max_results: Optional[int] = None) -> List[Dict[str, str]]:
+ def images(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, size: Optional[str] = None, color: Optional[str] = None, type_image: Optional[str] = None, layout: Optional[str] = None, license_image: Optional[str] = None, max_results: Optional[int] = None) -> List[ImagesResult]:
search = DuckDuckGoImages()
return search.run(keywords, region, safesearch, timelimit, size, color, type_image, layout, license_image, max_results)
- def videos(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, resolution: Optional[str] = None, duration: Optional[str] = None, license_videos: Optional[str] = None, max_results: Optional[int] = None) -> List[Dict[str, str]]:
+ def videos(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, resolution: Optional[str] = None, duration: Optional[str] = None, license_videos: Optional[str] = None, max_results: Optional[int] = None) -> List[VideosResult]:
search = DuckDuckGoVideos()
return search.run(keywords, region, safesearch, timelimit, resolution, duration, license_videos, max_results)
- def news(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, max_results: Optional[int] = None) -> List[Dict[str, str]]:
+ def news(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, max_results: Optional[int] = None) -> List[NewsResult]:
search = DuckDuckGoNews()
return search.run(keywords, region, safesearch, timelimit, max_results)
@@ -37,11 +40,11 @@ def answers(self, keywords: str) -> List[Dict[str, str]]:
search = DuckDuckGoAnswers()
return search.run(keywords)
- def suggestions(self, keywords: str, region: str = "wt-wt") -> List[str]:
+ def suggestions(self, keywords: str, region: str = "wt-wt") -> List[Dict[str, str]]:
search = DuckDuckGoSuggestions()
return search.run(keywords, region)
- def maps(self, keywords: str, place: Optional[str] = None, street: Optional[str] = None, city: Optional[str] = None, county: Optional[str] = None, state: Optional[str] = None, country: Optional[str] = None, postalcode: Optional[str] = None, latitude: Optional[str] = None, longitude: Optional[str] = None, radius: int = 0, max_results: Optional[int] = None) -> List[Dict[str, str]]:
+ def maps(self, keywords: str, place: Optional[str] = None, street: Optional[str] = None, city: Optional[str] = None, county: Optional[str] = None, state: Optional[str] = None, country: Optional[str] = None, postalcode: Optional[str] = None, latitude: Optional[str] = None, longitude: Optional[str] = None, radius: int = 0, max_results: Optional[int] = None) -> List[Dict[str, Any]]:
search = DuckDuckGoMaps()
return search.run(keywords, place, street, city, county, state, country, postalcode, latitude, longitude, radius, max_results)
@@ -49,6 +52,6 @@ def translate(self, keywords: str, from_lang: Optional[str] = None, to_lang: str
search = DuckDuckGoTranslate()
return search.run(keywords, from_lang, to_lang)
- def weather(self, keywords: str) -> List[Dict[str, str]]:
+ def weather(self, keywords: str) -> Dict[str, Any]:
search = DuckDuckGoWeather()
return search.run(keywords)
diff --git a/webscout/search/engines/__init__.py b/webscout/search/engines/__init__.py
index aae7877d..5be42820 100644
--- a/webscout/search/engines/__init__.py
+++ b/webscout/search/engines/__init__.py
@@ -2,33 +2,33 @@
from __future__ import annotations
+from ..base import BaseSearchEngine
+from .bing import BingBase, BingImagesSearch, BingNewsSearch, BingSuggestionsSearch, BingTextSearch
from .brave import Brave
-from .mojeek import Mojeek
-from .wikipedia import Wikipedia
-from .yandex import Yandex
-from .bing import BingBase, BingTextSearch, BingImagesSearch, BingNewsSearch, BingSuggestionsSearch
from .duckduckgo import (
+ DuckDuckGoAnswers,
DuckDuckGoBase,
- DuckDuckGoTextSearch,
DuckDuckGoImages,
- DuckDuckGoVideos,
+ DuckDuckGoMaps,
DuckDuckGoNews,
- DuckDuckGoAnswers,
DuckDuckGoSuggestions,
- DuckDuckGoMaps,
+ DuckDuckGoTextSearch,
DuckDuckGoTranslate,
+ DuckDuckGoVideos,
DuckDuckGoWeather,
)
-from .yep import YepBase, YepTextSearch, YepImages, YepSuggestions
+from .mojeek import Mojeek
+from .wikipedia import Wikipedia
from .yahoo import (
- YahooSearchEngine,
- YahooText,
YahooImages,
- YahooVideos,
YahooNews,
+ YahooSearchEngine,
YahooSuggestions,
+ YahooText,
+ YahooVideos,
)
-from ..base import BaseSearchEngine
+from .yandex import Yandex
+from .yep import YepBase, YepImages, YepSuggestions, YepTextSearch
# Engine categories mapping
ENGINES = {
diff --git a/webscout/search/engines/bing/__init__.py b/webscout/search/engines/bing/__init__.py
index fe7980c5..6447e62f 100644
--- a/webscout/search/engines/bing/__init__.py
+++ b/webscout/search/engines/bing/__init__.py
@@ -12,4 +12,4 @@
"BingImagesSearch",
"BingNewsSearch",
"BingSuggestionsSearch",
-]
\ No newline at end of file
+]
diff --git a/webscout/search/engines/bing/base.py b/webscout/search/engines/bing/base.py
index 75dcf6e5..0caaca41 100644
--- a/webscout/search/engines/bing/base.py
+++ b/webscout/search/engines/bing/base.py
@@ -2,9 +2,10 @@
from __future__ import annotations
-from ....litagent import LitAgent
from curl_cffi.requests import Session
+from ....litagent import LitAgent
+
class BingBase:
"""Base class for Bing search engines."""
@@ -30,4 +31,4 @@ def __init__(
timeout=timeout,
impersonate=impersonate,
)
- self.session.headers.update(LitAgent().generate_fingerprint())
\ No newline at end of file
+ self.session.headers.update(LitAgent().generate_fingerprint())
diff --git a/webscout/search/engines/bing/images.py b/webscout/search/engines/bing/images.py
index 6aa15cdc..8004e265 100644
--- a/webscout/search/engines/bing/images.py
+++ b/webscout/search/engines/bing/images.py
@@ -2,21 +2,22 @@
from __future__ import annotations
+from time import sleep
from typing import List
from urllib.parse import urlencode
-from time import sleep
-from .base import BingBase
from webscout.scout import Scout
from webscout.search.results import ImagesResult
+from .base import BingBase
+
class BingImagesSearch(BingBase):
name = "bing"
category = "images"
def run(self, *args, **kwargs) -> List[ImagesResult]:
keywords = args[0] if args else kwargs.get("keywords")
- region = args[1] if len(args) > 1 else kwargs.get("region", "us")
+ args[1] if len(args) > 1 else kwargs.get("region", "us")
safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10)
@@ -31,7 +32,7 @@ def run(self, *args, **kwargs) -> List[ImagesResult]:
"moderate": "Moderate",
"off": "Off"
}
- safe = safe_map.get(safesearch.lower(), "Moderate")
+ safe_map.get(safesearch.lower(), "Moderate")
# Bing images URL
url = f"{self.base_url}/images/async"
@@ -85,7 +86,7 @@ def run(self, *args, **kwargs) -> List[ImagesResult]:
m_data = json.loads(m_attr)
image_url = m_data.get('murl', src)
thumbnail = m_data.get('turl', src)
- except:
+ except Exception:
pass
source = ''
@@ -110,4 +111,4 @@ def run(self, *args, **kwargs) -> List[ImagesResult]:
if self.sleep_interval:
sleep(self.sleep_interval)
- return results[:max_results]
\ No newline at end of file
+ return results[:max_results]
diff --git a/webscout/search/engines/bing/news.py b/webscout/search/engines/bing/news.py
index 40278e91..4e281a35 100644
--- a/webscout/search/engines/bing/news.py
+++ b/webscout/search/engines/bing/news.py
@@ -2,14 +2,15 @@
from __future__ import annotations
+from time import sleep
from typing import List
from urllib.parse import urlencode
-from time import sleep
-from .base import BingBase
from webscout.scout import Scout
from webscout.search.results import NewsResult
+from .base import BingBase
+
class BingNewsSearch(BingBase):
name = "bing"
@@ -31,7 +32,7 @@ def run(self, *args, **kwargs) -> List[NewsResult]:
"moderate": "Moderate",
"off": "Off"
}
- safe = safe_map.get(safesearch.lower(), "Moderate")
+ safe_map.get(safesearch.lower(), "Moderate")
# Bing news URL
url = f"{self.base_url}/news/infinitescrollajax"
@@ -92,4 +93,4 @@ def run(self, *args, **kwargs) -> List[NewsResult]:
if self.sleep_interval:
sleep(self.sleep_interval)
- return results[:max_results]
\ No newline at end of file
+ return results[:max_results]
diff --git a/webscout/search/engines/bing/suggestions.py b/webscout/search/engines/bing/suggestions.py
index 4126bf75..11f83744 100644
--- a/webscout/search/engines/bing/suggestions.py
+++ b/webscout/search/engines/bing/suggestions.py
@@ -33,4 +33,4 @@ def run(self, *args, **kwargs) -> List[str]:
return data[1]
return []
except Exception as e:
- raise Exception(f"Failed to fetch suggestions: {str(e)}")
\ No newline at end of file
+ raise Exception(f"Failed to fetch suggestions: {str(e)}")
diff --git a/webscout/search/engines/bing/text.py b/webscout/search/engines/bing/text.py
index 96fc6538..68fccc6f 100644
--- a/webscout/search/engines/bing/text.py
+++ b/webscout/search/engines/bing/text.py
@@ -2,21 +2,21 @@
from __future__ import annotations
-from typing import List
-from urllib.parse import urlencode
from time import sleep
+from typing import List
-from .base import BingBase
from webscout.scout import Scout
from webscout.search.results import TextResult
+from .base import BingBase
+
class BingTextSearch(BingBase):
name = "bing"
category = "text"
def run(self, *args, **kwargs) -> List[TextResult]:
keywords = args[0] if args else kwargs.get("keywords")
- region = args[1] if len(args) > 1 else kwargs.get("region", "us")
+ args[1] if len(args) > 1 else kwargs.get("region", "us")
safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10)
unique = kwargs.get("unique", True)
@@ -32,7 +32,7 @@ def run(self, *args, **kwargs) -> List[TextResult]:
"moderate": "Moderate",
"off": "Off"
}
- safe = safe_map.get(safesearch.lower(), "Moderate")
+ safe_map.get(safesearch.lower(), "Moderate")
fetched_results = []
fetched_links = set()
@@ -84,7 +84,7 @@ def fetch_page(url):
import base64
decoded = base64.urlsafe_b64decode(encoded_url).decode()
href = decoded
- except:
+ except Exception:
pass
if unique and href in fetched_links:
@@ -106,4 +106,4 @@ def fetch_page(url):
if self.sleep_interval:
sleep(self.sleep_interval)
- return fetched_results[:max_results]
\ No newline at end of file
+ return fetched_results[:max_results]
diff --git a/webscout/search/engines/brave.py b/webscout/search/engines/brave.py
index 6ff6235b..24f93f1b 100644
--- a/webscout/search/engines/brave.py
+++ b/webscout/search/engines/brave.py
@@ -44,13 +44,13 @@ def build_payload(
def run(self, *args, **kwargs) -> list[TextResult]:
"""Run text search on Brave.
-
+
Args:
keywords: Search query.
region: Region code.
safesearch: Safe search level.
max_results: Maximum number of results (ignored for now).
-
+
Returns:
List of TextResult objects.
"""
@@ -58,7 +58,7 @@ def run(self, *args, **kwargs) -> list[TextResult]:
region = args[1] if len(args) > 1 else kwargs.get("region", "us-en")
safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
max_results = args[3] if len(args) > 3 else kwargs.get("max_results")
-
+
results = self.search(query=keywords, region=region, safesearch=safesearch)
if results and max_results:
results = results[:max_results]
diff --git a/webscout/search/engines/duckduckgo/answers.py b/webscout/search/engines/duckduckgo/answers.py
index 79227921..110cd955 100644
--- a/webscout/search/engines/duckduckgo/answers.py
+++ b/webscout/search/engines/duckduckgo/answers.py
@@ -2,22 +2,21 @@
from __future__ import annotations
-from ....exceptions import WebscoutE
from .base import DuckDuckGoBase
class DuckDuckGoAnswers(DuckDuckGoBase):
"""DuckDuckGo instant answers."""
-
+
name = "duckduckgo"
category = "answers"
-
+
def run(self, *args, **kwargs) -> list[dict[str, str]]:
"""Get instant answers from DuckDuckGo.
-
+
Args:
keywords: Search query.
-
+
Returns:
List of answer dictionaries.
"""
diff --git a/webscout/search/engines/duckduckgo/base.py b/webscout/search/engines/duckduckgo/base.py
index adb301a5..2fd3d002 100644
--- a/webscout/search/engines/duckduckgo/base.py
+++ b/webscout/search/engines/duckduckgo/base.py
@@ -11,7 +11,7 @@
from typing import Any
try:
- import trio # type: ignore
+ import trio # type: ignore
except ImportError:
pass
@@ -25,13 +25,13 @@
LXML_AVAILABLE = False
from ....exceptions import RatelimitE, TimeoutE, WebscoutE
+from ....litagent import LitAgent
from ....utils import (
_extract_vqd,
_normalize,
_normalize_url,
json_loads,
)
-from ....litagent import LitAgent
class DuckDuckGoBase:
@@ -66,7 +66,7 @@ def __init__(
"""
ddgs_proxy: str | None = os.environ.get("DDGS_PROXY")
self.proxy: str | None = ddgs_proxy if ddgs_proxy else proxy
-
+
if not proxy and proxies:
self.proxy = proxies.get("http") or proxies.get("https") if isinstance(proxies, dict) else proxies
@@ -89,7 +89,7 @@ def __init__(
)
self.timeout = timeout
self.sleep_timestamp = 0.0
-
+
# Utility methods
self.cycle = cycle
self.islice = islice
@@ -99,7 +99,7 @@ def parser(self) -> Any:
"""Get HTML parser."""
if not LXML_AVAILABLE:
raise ImportError("lxml is required for HTML parsing")
-
+
class Parser:
def __init__(self):
self.lhtml_parser = LHTMLParser(
@@ -109,10 +109,10 @@ def __init__(self):
collect_ids=False
)
self.etree = __import__('lxml.etree', fromlist=['Element'])
-
+
def fromstring(self, html: bytes | str) -> Any:
return document_fromstring(html, parser=self.lhtml_parser)
-
+
return Parser()
def _sleep(self, sleeptime: float = 0.75) -> None:
@@ -162,7 +162,7 @@ def _get_url(
if "time" in str(ex).lower():
raise TimeoutE(f"{url} {type(ex).__name__}: {ex}") from ex
raise WebscoutE(f"{url} {type(ex).__name__}: {ex}") from ex
-
+
if resp.status_code == 200:
return resp
elif resp.status_code in (202, 301, 403, 400, 429, 418):
diff --git a/webscout/search/engines/duckduckgo/images.py b/webscout/search/engines/duckduckgo/images.py
index 27de59fe..f6d1f9a7 100644
--- a/webscout/search/engines/duckduckgo/images.py
+++ b/webscout/search/engines/duckduckgo/images.py
@@ -2,20 +2,19 @@
from __future__ import annotations
-from ....exceptions import WebscoutE
from ....search.results import ImagesResult
from .base import DuckDuckGoBase
class DuckDuckGoImages(DuckDuckGoBase):
"""DuckDuckGo image search."""
-
+
name = "duckduckgo"
category = "images"
-
+
def run(self, *args, **kwargs) -> list[ImagesResult]:
"""Perform image search on DuckDuckGo.
-
+
Args:
keywords: Search query.
region: Region code.
@@ -27,7 +26,7 @@ def run(self, *args, **kwargs) -> list[ImagesResult]:
layout: Square, Tall, Wide.
license_image: any, Public, Share, etc.
max_results: Maximum number of results.
-
+
Returns:
List of ImagesResult objects.
"""
diff --git a/webscout/search/engines/duckduckgo/maps.py b/webscout/search/engines/duckduckgo/maps.py
index 12531a83..e288cc8b 100644
--- a/webscout/search/engines/duckduckgo/maps.py
+++ b/webscout/search/engines/duckduckgo/maps.py
@@ -17,7 +17,7 @@ def _calculate_distance(self, lat_t: Decimal, lon_l: Decimal, lat_b: Decimal, lo
lon_l_f = float(lon_l)
lat_b_f = float(lat_b)
lon_r_f = float(lon_r)
-
+
# Calculate Euclidean distance
distance = sqrt((lat_t_f - lat_b_f) ** 2 + (lon_r_f - lon_l_f) ** 2)
return distance
diff --git a/webscout/search/engines/duckduckgo/news.py b/webscout/search/engines/duckduckgo/news.py
index 2a8ac544..5307e050 100644
--- a/webscout/search/engines/duckduckgo/news.py
+++ b/webscout/search/engines/duckduckgo/news.py
@@ -2,7 +2,6 @@
from datetime import datetime, timezone
-from ....exceptions import WebscoutE
from ....search.results import NewsResult
from .base import DuckDuckGoBase
diff --git a/webscout/search/engines/duckduckgo/suggestions.py b/webscout/search/engines/duckduckgo/suggestions.py
index 607eba8c..4d12b6fb 100644
--- a/webscout/search/engines/duckduckgo/suggestions.py
+++ b/webscout/search/engines/duckduckgo/suggestions.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-from ....exceptions import WebscoutE
from .base import DuckDuckGoBase
diff --git a/webscout/search/engines/duckduckgo/text.py b/webscout/search/engines/duckduckgo/text.py
index e3b2fc9c..8e23374a 100644
--- a/webscout/search/engines/duckduckgo/text.py
+++ b/webscout/search/engines/duckduckgo/text.py
@@ -12,13 +12,13 @@
class DuckDuckGoTextSearch(DuckDuckGoBase):
"""DuckDuckGo text/web search."""
-
+
name = "duckduckgo"
category = "text"
-
+
def run(self, *args, **kwargs) -> list[TextResult]:
"""Perform text search on DuckDuckGo.
-
+
Args:
keywords: Search query.
region: Region code (e.g., wt-wt, us-en).
@@ -26,13 +26,13 @@ def run(self, *args, **kwargs) -> list[TextResult]:
timelimit: d, w, m, or y.
backend: html, lite, or auto.
max_results: Maximum number of results.
-
+
Returns:
List of TextResult objects.
"""
keywords = args[0] if args else kwargs.get("keywords")
region = args[1] if len(args) > 1 else kwargs.get("region", "wt-wt")
- safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+ args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
timelimit = args[3] if len(args) > 3 else kwargs.get("timelimit")
backend = args[4] if len(args) > 4 else kwargs.get("backend", "auto")
max_results = args[5] if len(args) > 5 else kwargs.get("max_results")
diff --git a/webscout/search/engines/duckduckgo/translate.py b/webscout/search/engines/duckduckgo/translate.py
index d4c827e7..e0401a51 100644
--- a/webscout/search/engines/duckduckgo/translate.py
+++ b/webscout/search/engines/duckduckgo/translate.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-from ....exceptions import WebscoutE
from .base import DuckDuckGoBase
diff --git a/webscout/search/engines/duckduckgo/videos.py b/webscout/search/engines/duckduckgo/videos.py
index 211227c2..99c23343 100644
--- a/webscout/search/engines/duckduckgo/videos.py
+++ b/webscout/search/engines/duckduckgo/videos.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-from ....exceptions import WebscoutE
from ....search.results import VideosResult
from .base import DuckDuckGoBase
diff --git a/webscout/search/engines/mojeek.py b/webscout/search/engines/mojeek.py
index 8c9c9259..fde1d694 100644
--- a/webscout/search/engines/mojeek.py
+++ b/webscout/search/engines/mojeek.py
@@ -38,13 +38,13 @@ def build_payload(
def run(self, *args, **kwargs) -> list[TextResult]:
"""Run text search on Mojeek.
-
+
Args:
keywords: Search query.
region: Region code.
safesearch: Safe search level.
max_results: Maximum number of results (ignored for now).
-
+
Returns:
List of TextResult objects.
"""
@@ -52,7 +52,7 @@ def run(self, *args, **kwargs) -> list[TextResult]:
region = args[1] if len(args) > 1 else kwargs.get("region", "us-en")
safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
max_results = args[3] if len(args) > 3 else kwargs.get("max_results")
-
+
results = self.search(query=keywords, region=region, safesearch=safesearch)
if results and max_results:
results = results[:max_results]
diff --git a/webscout/search/engines/wikipedia.py b/webscout/search/engines/wikipedia.py
index 22d496e8..38bcd003 100644
--- a/webscout/search/engines/wikipedia.py
+++ b/webscout/search/engines/wikipedia.py
@@ -2,13 +2,12 @@
from __future__ import annotations
-from litprinter import ic
from typing import Any
from urllib.parse import quote
+from ...utils import json_loads
from ..base import BaseSearchEngine
from ..results import TextResult
-from ...utils import json_loads
class Wikipedia(BaseSearchEngine[TextResult]):
@@ -40,28 +39,28 @@ def extract_results(self, html_text: str) -> list[TextResult]:
json_data = json_loads(html_text)
if not json_data or len(json_data) < 4:
return []
-
+
results = []
titles, descriptions, urls = json_data[1], json_data[2], json_data[3]
-
+
for title, description, url in zip(titles, descriptions, urls):
result = TextResult()
result.title = title
result.body = description
result.href = url
results.append(result)
-
+
return results
def run(self, *args, **kwargs) -> list[TextResult]:
"""Run text search on Wikipedia.
-
+
Args:
keywords: Search query.
region: Region code.
safesearch: Safe search level (ignored).
max_results: Maximum number of results.
-
+
Returns:
List of TextResult objects.
"""
@@ -69,7 +68,7 @@ def run(self, *args, **kwargs) -> list[TextResult]:
region = args[1] if len(args) > 1 else kwargs.get("region", "en-us")
safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
max_results = args[3] if len(args) > 3 else kwargs.get("max_results")
-
+
results = self.search(query=keywords, region=region, safesearch=safesearch)
if results and max_results:
results = results[:max_results]
diff --git a/webscout/search/engines/yahoo/__init__.py b/webscout/search/engines/yahoo/__init__.py
index aed2f9d8..2212c08b 100644
--- a/webscout/search/engines/yahoo/__init__.py
+++ b/webscout/search/engines/yahoo/__init__.py
@@ -15,11 +15,11 @@
Example:
>>> from webscout.search.engines.yahoo import YahooText
- >>>
+ >>>
>>> # Search with automatic pagination
>>> searcher = YahooText()
>>> results = searcher.search("python programming", max_results=50)
- >>>
+ >>>
>>> for result in results:
... print(f"{result.title}: {result.url}")
"""
diff --git a/webscout/search/engines/yahoo/answers.py b/webscout/search/engines/yahoo/answers.py
index 0d429e5b..ade1c576 100644
--- a/webscout/search/engines/yahoo/answers.py
+++ b/webscout/search/engines/yahoo/answers.py
@@ -13,4 +13,4 @@ def run(self, *args, **kwargs) -> list[dict[str, str]]:
Not supported.
"""
- raise NotImplementedError("Yahoo does not support instant answers")
\ No newline at end of file
+ raise NotImplementedError("Yahoo does not support instant answers")
diff --git a/webscout/search/engines/yahoo/base.py b/webscout/search/engines/yahoo/base.py
index a9342ddf..10d7dac0 100644
--- a/webscout/search/engines/yahoo/base.py
+++ b/webscout/search/engines/yahoo/base.py
@@ -3,7 +3,7 @@
from __future__ import annotations
from secrets import token_urlsafe
-from typing import Any, Generic, TypeVar
+from typing import Generic, TypeVar
from ...base import BaseSearchEngine
@@ -11,22 +11,22 @@
class YahooSearchEngine(BaseSearchEngine[T], Generic[T]):
"""Base class for Yahoo search engines.
-
+
Yahoo search is powered by Bing but has its own interface.
All Yahoo searches use dynamic URLs with tokens for tracking.
"""
provider = "yahoo"
_base_url = "https://search.yahoo.com"
-
+
def generate_ylt_token(self) -> str:
"""Generate Yahoo _ylt tracking token."""
return token_urlsafe(24 * 3 // 4)
-
+
def generate_ylu_token(self) -> str:
"""Generate Yahoo _ylu tracking token."""
return token_urlsafe(47 * 3 // 4)
-
+
def build_search_url(self, base_path: str) -> str:
"""Build search URL with tracking tokens."""
ylt = self.generate_ylt_token()
diff --git a/webscout/search/engines/yahoo/images.py b/webscout/search/engines/yahoo/images.py
index 9403626d..10fcbeb1 100644
--- a/webscout/search/engines/yahoo/images.py
+++ b/webscout/search/engines/yahoo/images.py
@@ -4,15 +4,14 @@
from collections.abc import Mapping
from typing import Any
-from urllib.parse import urljoin
-from .base import YahooSearchEngine
from ...results import ImagesResult
+from .base import YahooSearchEngine
class YahooImages(YahooSearchEngine[ImagesResult]):
"""Yahoo image search engine with filter support.
-
+
Features:
- Size filters (small, medium, large, wallpaper)
- Color filters (color, bw, red, orange, yellow, etc.)
@@ -20,7 +19,7 @@ class YahooImages(YahooSearchEngine[ImagesResult]):
- Layout filters (square, wide, tall)
- Time filters
- Pagination support
-
+
Note: Yahoo does not support reverse image search (searching by image upload/URL).
For reverse image search functionality, use Google Images or Bing Images instead.
"""
@@ -38,7 +37,7 @@ class YahooImages(YahooSearchEngine[ImagesResult]):
items_xpath = "//li[contains(@class, 'ld')]"
elements_xpath: Mapping[str, str] = {
"title": "@data",
- "image": "@data",
+ "image": "@data",
"thumbnail": "@data",
"url": "@data",
"source": "@data",
@@ -49,7 +48,7 @@ class YahooImages(YahooSearchEngine[ImagesResult]):
# Filter mappings
SIZE_FILTERS = {
"small": "small",
- "medium": "medium",
+ "medium": "medium",
"large": "large",
"wallpaper": "wallpaper",
"all": "",
@@ -99,7 +98,7 @@ def build_payload(
**kwargs: Any,
) -> dict[str, Any]:
"""Build image search payload with filters.
-
+
Args:
query: Search query
region: Region code
@@ -112,7 +111,7 @@ def build_payload(
- type: Image type filter
- layout: Layout/aspect ratio filter
- license: Usage rights filter
-
+
Returns:
Query parameters dictionary
"""
@@ -169,40 +168,40 @@ def build_payload(
def post_extract_results(self, results: list[ImagesResult]) -> list[ImagesResult]:
"""Post-process image results to parse JSON data.
-
+
Args:
results: Raw extracted results
-
+
Returns:
Cleaned results with proper URLs and metadata
"""
import json
from urllib.parse import unquote
-
+
cleaned_results = []
-
+
for result in results:
# Parse JSON data from the data attribute
if result.title and result.title.startswith('{'):
try:
data = json.loads(result.title)
-
+
# Extract title
result.title = data.get('desc', '') or data.get('tit', '')
-
+
# Extract URLs
result.url = data.get('rurl', '')
result.thumbnail = data.get('turl', '')
result.image = data.get('turlL', '') or data.get('turl', '')
-
+
# Extract dimensions
result.width = int(data.get('imgW', 0))
result.height = int(data.get('imgH', 0))
-
+
except (json.JSONDecodeError, KeyError, ValueError):
# If JSON parsing fails, keep original data
pass
-
+
# Clean URLs if they exist
if result.url:
result.url = unquote(result.url)
@@ -210,9 +209,9 @@ def post_extract_results(self, results: list[ImagesResult]) -> list[ImagesResult
result.image = unquote(result.image)
if result.thumbnail:
result.thumbnail = unquote(result.thumbnail)
-
+
cleaned_results.append(result)
-
+
return cleaned_results
def search(
@@ -226,7 +225,7 @@ def search(
**kwargs: Any,
) -> list[ImagesResult] | None:
"""Search Yahoo Images with pagination.
-
+
Args:
query: Image search query
region: Region code
@@ -235,14 +234,14 @@ def search(
page: Starting page
max_results: Maximum results to return
**kwargs: Additional filters (size, color, type, layout)
-
+
Returns:
List of ImageResult objects
"""
results = []
current_page = page
max_pages = kwargs.get("max_pages", 5)
-
+
while current_page <= max_pages:
payload = self.build_payload(
query=query,
@@ -252,29 +251,29 @@ def search(
page=current_page,
**kwargs
)
-
+
html_text = self.request(self.search_method, self.search_url, params=payload)
if not html_text:
break
-
+
html_text = self.pre_process_html(html_text)
page_results = self.extract_results(html_text)
-
+
if not page_results:
break
-
+
results.extend(page_results)
-
+
if max_results and len(results) >= max_results:
break
-
+
current_page += 1
-
+
results = self.post_extract_results(results)
-
+
if max_results:
results = results[:max_results]
-
+
return results if results else None
def run(
@@ -291,7 +290,7 @@ def run(
max_results: int | None = None,
) -> list[dict[str, str]]:
"""Run image search and return results as dictionaries.
-
+
Args:
keywords: Search query.
region: Region code.
@@ -303,7 +302,7 @@ def run(
layout: Layout filter.
license_image: License filter.
max_results: Maximum number of results.
-
+
Returns:
List of image result dictionaries.
"""
diff --git a/webscout/search/engines/yahoo/maps.py b/webscout/search/engines/yahoo/maps.py
index 5aa28c5b..cf611464 100644
--- a/webscout/search/engines/yahoo/maps.py
+++ b/webscout/search/engines/yahoo/maps.py
@@ -13,4 +13,4 @@ def run(self, *args, **kwargs) -> list[dict[str, str]]:
Not supported.
"""
- raise NotImplementedError("Yahoo does not support maps search")
\ No newline at end of file
+ raise NotImplementedError("Yahoo does not support maps search")
diff --git a/webscout/search/engines/yahoo/news.py b/webscout/search/engines/yahoo/news.py
index 4b1068bc..7fa34a95 100644
--- a/webscout/search/engines/yahoo/news.py
+++ b/webscout/search/engines/yahoo/news.py
@@ -6,47 +6,47 @@
from secrets import token_urlsafe
from typing import Any
-from .base import YahooSearchEngine
from ...results import NewsResult
+from .base import YahooSearchEngine
def extract_image(u: str) -> str:
"""Sanitize image URL.
-
+
Args:
u: Image URL
-
+
Returns:
Cleaned URL or empty string
"""
if not u:
return ""
-
+
# Skip data URIs
if u.startswith("data:image"):
return ""
-
+
return u
def extract_source(s: str) -> str:
"""Remove ' via Yahoo' from source string.
-
+
Args:
s: Source string
-
+
Returns:
Cleaned source name
"""
if not s:
return s
-
+
return s.replace(" via Yahoo", "").replace(" - Yahoo", "").strip()
class YahooNews(YahooSearchEngine[NewsResult]):
"""Yahoo news search engine with advanced filtering.
-
+
Features:
- Time-based filtering
- Category filtering
@@ -82,7 +82,7 @@ def build_payload(
**kwargs: Any,
) -> dict[str, Any]:
"""Build news search payload.
-
+
Args:
query: Search query
region: Region code
@@ -90,7 +90,7 @@ def build_payload(
timelimit: Time filter (d, w, m)
page: Page number
**kwargs: Additional parameters
-
+
Returns:
Query parameters dictionary
"""
@@ -100,17 +100,17 @@ def build_payload(
f";_ylt={token_urlsafe(24 * 3 // 4)}"
f";_ylu={token_urlsafe(47 * 3 // 4)}"
)
-
+
payload = {
"p": query,
"ei": "UTF-8",
}
-
+
# Pagination - Yahoo news uses 'b' parameter
if page > 1:
# Each page shows approximately 10 articles
payload["b"] = f"{(page - 1) * 10 + 1}"
-
+
# Time filter
if timelimit:
time_map = {
@@ -120,35 +120,35 @@ def build_payload(
}
if timelimit in time_map:
payload["btf"] = time_map[timelimit]
-
+
# Additional filters
if "category" in kwargs:
payload["category"] = kwargs["category"]
-
+
if "sort" in kwargs:
# Sort by relevance or date
payload["sort"] = kwargs["sort"]
-
+
return payload
def post_extract_results(self, results: list[NewsResult]) -> list[NewsResult]:
"""Post-process news results.
-
+
Args:
results: Raw extracted results
-
+
Returns:
Cleaned news results
"""
cleaned_results = []
-
+
for result in results:
# Clean image URL
result.image = extract_image(result.image)
-
+
# Clean source name
result.source = extract_source(result.source)
-
+
# Extract URL from redirect
if result.url and "/RU=" in result.url:
from urllib.parse import unquote
@@ -157,11 +157,11 @@ def post_extract_results(self, results: list[NewsResult]) -> list[NewsResult]:
if end == -1:
end = len(result.url)
result.url = unquote(result.url[start:end])
-
+
# Filter out results without essential fields
if result.title and result.url:
cleaned_results.append(result)
-
+
return cleaned_results
def search(
@@ -175,7 +175,7 @@ def search(
**kwargs: Any,
) -> list[NewsResult] | None:
"""Search Yahoo News with pagination.
-
+
Args:
query: News search query
region: Region code
@@ -184,14 +184,14 @@ def search(
page: Starting page
max_results: Maximum results to return
**kwargs: Additional parameters (category, sort)
-
+
Returns:
List of NewsResult objects
"""
results = []
current_page = page
max_pages = kwargs.get("max_pages", 10)
-
+
while current_page <= max_pages:
payload = self.build_payload(
query=query,
@@ -201,29 +201,29 @@ def search(
page=current_page,
**kwargs
)
-
+
html_text = self.request(self.search_method, self.search_url, params=payload)
if not html_text:
break
-
+
html_text = self.pre_process_html(html_text)
page_results = self.extract_results(html_text)
-
+
if not page_results:
break
-
+
results.extend(page_results)
-
+
if max_results and len(results) >= max_results:
break
-
+
current_page += 1
-
+
results = self.post_extract_results(results)
-
+
if max_results:
results = results[:max_results]
-
+
return results if results else None
def run(
@@ -235,14 +235,14 @@ def run(
max_results: int | None = None,
) -> list[dict[str, str]]:
"""Run news search and return results as dictionaries.
-
+
Args:
keywords: Search query.
region: Region code.
safesearch: Safe search level.
timelimit: Time filter.
max_results: Maximum number of results.
-
+
Returns:
List of news result dictionaries.
"""
diff --git a/webscout/search/engines/yahoo/suggestions.py b/webscout/search/engines/yahoo/suggestions.py
index 42c5b3ea..905d32f2 100644
--- a/webscout/search/engines/yahoo/suggestions.py
+++ b/webscout/search/engines/yahoo/suggestions.py
@@ -10,7 +10,7 @@
class YahooSuggestions(YahooSearchEngine[str]):
"""Yahoo search suggestions engine.
-
+
Provides autocomplete suggestions as you type.
"""
@@ -30,7 +30,7 @@ def build_payload(
**kwargs: Any,
) -> dict[str, Any]:
"""Build suggestions payload.
-
+
Args:
query: Partial search query
region: Region code
@@ -38,7 +38,7 @@ def build_payload(
timelimit: Time limit (unused)
page: Page number (unused)
**kwargs: Additional parameters
-
+
Returns:
Query parameters
"""
@@ -47,21 +47,21 @@ def build_payload(
"output": "sd1",
"nresults": kwargs.get("max_suggestions", 10),
}
-
+
return payload
def extract_results(self, html_text: str) -> list[str]:
"""Extract suggestions from JSON response.
-
+
Args:
html_text: JSON response text
-
+
Returns:
List of suggestion strings
"""
try:
data = json.loads(html_text)
-
+
# Yahoo returns suggestions in 'r' key
if "r" in data and isinstance(data["r"], list):
suggestions = []
@@ -71,7 +71,7 @@ def extract_results(self, html_text: str) -> list[str]:
elif isinstance(item, str):
suggestions.append(item)
return suggestions
-
+
return []
except (json.JSONDecodeError, KeyError, TypeError):
return []
@@ -87,7 +87,7 @@ def search(
**kwargs: Any,
) -> list[str] | None:
"""Get search suggestions for a query.
-
+
Args:
query: Partial search query
region: Region code
@@ -96,13 +96,13 @@ def search(
page: Page number
max_results: Maximum suggestions
**kwargs: Additional parameters
-
+
Returns:
List of suggestion strings
"""
if max_results:
kwargs["max_suggestions"] = max_results
-
+
payload = self.build_payload(
query=query,
region=region,
@@ -111,25 +111,25 @@ def search(
page=page,
**kwargs
)
-
+
response = self.request(self.search_method, self.search_url, params=payload)
if not response:
return None
-
+
suggestions = self.extract_results(response)
-
+
if max_results:
suggestions = suggestions[:max_results]
-
+
return suggestions if suggestions else None
def run(self, keywords: str, region: str = "us-en") -> list[str]:
"""Run suggestions search and return results.
-
+
Args:
keywords: Search query.
region: Region code.
-
+
Returns:
List of suggestion strings.
"""
diff --git a/webscout/search/engines/yahoo/text.py b/webscout/search/engines/yahoo/text.py
index 3ccab5fa..a7c1f22a 100644
--- a/webscout/search/engines/yahoo/text.py
+++ b/webscout/search/engines/yahoo/text.py
@@ -4,21 +4,21 @@
from collections.abc import Mapping
from typing import Any
-from urllib.parse import unquote_plus, urljoin
+from urllib.parse import unquote_plus
-from .base import YahooSearchEngine
from ...results import TextResult
+from .base import YahooSearchEngine
def extract_url(u: str) -> str:
"""Extract and sanitize URL from Yahoo redirect.
-
+
Yahoo uses /RU= redirect URLs that need to be decoded.
Example: /url?sa=t&url=https%3A%2F%2Fexample.com
"""
if not u:
return u
-
+
# Handle /RU= redirect format
if "/RU=" in u:
start = u.find("/RU=") + 4
@@ -26,13 +26,13 @@ def extract_url(u: str) -> str:
if end == -1:
end = len(u)
return unquote_plus(u[start:end])
-
+
return u
class YahooText(YahooSearchEngine[TextResult]):
"""Yahoo text search engine with full pagination support.
-
+
Features:
- Multi-page navigation like a human
- Automatic next page detection
@@ -56,16 +56,16 @@ class YahooText(YahooSearchEngine[TextResult]):
}
def build_payload(
- self,
- query: str,
- region: str,
- safesearch: str,
- timelimit: str | None,
- page: int = 1,
+ self,
+ query: str,
+ region: str,
+ safesearch: str,
+ timelimit: str | None,
+ page: int = 1,
**kwargs: Any
) -> dict[str, Any]:
"""Build search payload for Yahoo.
-
+
Args:
query: Search query string
region: Region code (e.g., 'us-en')
@@ -73,7 +73,7 @@ def build_payload(
timelimit: Time limit filter (d=day, w=week, m=month)
page: Page number (1-indexed)
**kwargs: Additional parameters
-
+
Returns:
Dictionary of query parameters
"""
@@ -81,40 +81,40 @@ def build_payload(
"p": query,
"ei": "UTF-8",
}
-
+
# Pagination: Yahoo uses 'b' parameter for offset
# Page 1: no b parameter or b=1
# Page 2: b=8 (shows results 8-14)
# Page 3: b=15, etc.
if page > 1:
payload["b"] = f"{(page - 1) * 7 + 1}"
-
+
# Time filter
if timelimit:
payload["btf"] = timelimit
-
+
return payload
def post_extract_results(self, results: list[TextResult]) -> list[TextResult]:
"""Post-process and clean extracted results.
-
+
Args:
results: Raw extracted results
-
+
Returns:
Cleaned and filtered results
"""
cleaned_results = []
-
+
for result in results:
# Extract real URL from redirect
if result.href:
result.href = extract_url(result.href)
-
+
# Filter out empty results
if result.title and result.href:
cleaned_results.append(result)
-
+
return cleaned_results
def search(
@@ -128,10 +128,10 @@ def search(
**kwargs: Any,
) -> list[TextResult] | None:
"""Search Yahoo with automatic pagination like a human browser.
-
+
This method automatically follows pagination links to gather results
across multiple pages, similar to how a human would browse search results.
-
+
Args:
query: Search query string
region: Region code
@@ -140,14 +140,14 @@ def search(
page: Starting page number
max_results: Maximum number of results to return
**kwargs: Additional search parameters
-
+
Returns:
List of TextResult objects, or None if search fails
"""
results = []
current_page = page
max_pages = kwargs.get("max_pages", 10) # Limit to prevent infinite loops
-
+
while current_page <= max_pages:
# Build payload for current page
payload = self.build_payload(
@@ -158,45 +158,45 @@ def search(
page=current_page,
**kwargs
)
-
+
# Make request
html_text = self.request(self.search_method, self.search_url, params=payload)
if not html_text:
break
-
+
# Pre-process HTML
html_text = self.pre_process_html(html_text)
-
+
# Extract results from current page
page_results = self.extract_results(html_text)
if not page_results:
break
-
+
results.extend(page_results)
-
+
# Check if we have enough results
if max_results and len(results) >= max_results:
break
-
+
# Look for next page link
tree = self.extract_tree(html_text)
next_links = tree.xpath("//a[contains(text(), 'Next') or contains(@class, 'next')]/@href")
-
+
if not next_links:
# Try to find numbered page links
page_links = tree.xpath(f"//a[contains(text(), '{current_page + 1}')]/@href")
if not page_links:
break
-
+
current_page += 1
-
+
# Post-process all results
results = self.post_extract_results(results)
-
+
# Trim to max_results if specified
if max_results:
results = results[:max_results]
-
+
return results if results else None
def search_page(
@@ -209,7 +209,7 @@ def search_page(
**kwargs: Any,
) -> list[TextResult] | None:
"""Search a single page (for compatibility).
-
+
Args:
query: Search query
region: Region code
@@ -217,7 +217,7 @@ def search_page(
timelimit: Time filter
page: Page number
**kwargs: Additional parameters
-
+
Returns:
List of results from the specified page
"""
@@ -229,14 +229,14 @@ def search_page(
page=page,
**kwargs
)
-
+
html_text = self.request(self.search_method, self.search_url, params=payload)
if not html_text:
return None
-
+
html_text = self.pre_process_html(html_text)
results = self.extract_results(html_text)
-
+
return self.post_extract_results(results) if results else None
def run(
@@ -249,7 +249,7 @@ def run(
max_results: int | None = None,
) -> list[dict[str, str]]:
"""Run text search and return results as dictionaries.
-
+
Args:
keywords: Search query.
region: Region code.
@@ -257,7 +257,7 @@ def run(
timelimit: Time filter.
backend: Backend type (ignored for Yahoo).
max_results: Maximum number of results.
-
+
Returns:
List of search result dictionaries.
"""
diff --git a/webscout/search/engines/yahoo/translate.py b/webscout/search/engines/yahoo/translate.py
index 6e075383..973456f9 100644
--- a/webscout/search/engines/yahoo/translate.py
+++ b/webscout/search/engines/yahoo/translate.py
@@ -13,4 +13,4 @@ def run(self, *args, **kwargs) -> list[dict[str, str]]:
Not supported.
"""
- raise NotImplementedError("Yahoo does not support translation")
\ No newline at end of file
+ raise NotImplementedError("Yahoo does not support translation")
diff --git a/webscout/search/engines/yahoo/videos.py b/webscout/search/engines/yahoo/videos.py
index 5deb46bf..6c4378ae 100644
--- a/webscout/search/engines/yahoo/videos.py
+++ b/webscout/search/engines/yahoo/videos.py
@@ -6,13 +6,13 @@
from typing import Any
from urllib.parse import parse_qs, urlparse
-from .base import YahooSearchEngine
from ...results import VideosResult
+from .base import YahooSearchEngine
class YahooVideos(YahooSearchEngine[VideosResult]):
"""Yahoo video search engine with filters.
-
+
Features:
- Length filters (short, medium, long)
- Resolution filters (SD, HD, 4K)
@@ -72,7 +72,7 @@ def build_payload(
**kwargs: Any,
) -> dict[str, Any]:
"""Build video search payload.
-
+
Args:
query: Search query
region: Region code
@@ -83,7 +83,7 @@ def build_payload(
- length: Video length filter
- resolution: Video resolution filter
- source: Video source filter
-
+
Returns:
Query parameters dictionary
"""
@@ -138,20 +138,20 @@ def build_payload(
def extract_video_url(self, href: str) -> str:
"""Extract actual video URL from Yahoo redirect.
-
+
Args:
href: Yahoo redirect URL
-
+
Returns:
Actual video URL
"""
if not href:
return href
-
+
try:
# Parse the URL
parsed = urlparse(href)
-
+
# Check if it's a Yahoo redirect
if "r.search.yahoo.com" in parsed.netloc or "/RU=" in href:
# Extract the RU parameter
@@ -166,37 +166,37 @@ def extract_video_url(self, href: str) -> str:
query_params = parse_qs(parsed.query)
if "url" in query_params:
return query_params["url"][0]
-
+
return href
except Exception:
return href
def post_extract_results(self, results: list[VideosResult]) -> list[VideosResult]:
"""Post-process video results.
-
+
Args:
results: Raw extracted results
-
+
Returns:
Cleaned results
"""
cleaned_results = []
-
+
for result in results:
# Extract real URL
if result.url:
result.url = self.extract_video_url(result.url)
-
+
# Skip invalid results
if not result.url or not result.title:
continue
-
+
# Clean thumbnail URL
if result.thumbnail and result.thumbnail.startswith("data:"):
result.thumbnail = ""
-
+
cleaned_results.append(result)
-
+
return cleaned_results
def search(
@@ -210,7 +210,7 @@ def search(
**kwargs: Any,
) -> list[VideosResult] | None:
"""Search Yahoo Videos with pagination.
-
+
Args:
query: Video search query
region: Region code
@@ -219,14 +219,14 @@ def search(
page: Starting page
max_results: Maximum results
**kwargs: Additional filters (length, resolution, source)
-
+
Returns:
List of VideoResult objects
"""
results = []
current_page = page
max_pages = kwargs.get("max_pages", 5)
-
+
while current_page <= max_pages:
payload = self.build_payload(
query=query,
@@ -236,29 +236,29 @@ def search(
page=current_page,
**kwargs
)
-
+
html_text = self.request(self.search_method, self.search_url, params=payload)
if not html_text:
break
-
+
html_text = self.pre_process_html(html_text)
page_results = self.extract_results(html_text)
-
+
if not page_results:
break
-
+
results.extend(page_results)
-
+
if max_results and len(results) >= max_results:
break
-
+
current_page += 1
-
+
results = self.post_extract_results(results)
-
+
if max_results:
results = results[:max_results]
-
+
return results if results else None
def run(
@@ -273,7 +273,7 @@ def run(
max_results: int | None = None,
) -> list[dict[str, str]]:
"""Run video search and return results as dictionaries.
-
+
Args:
keywords: Search query.
region: Region code.
@@ -283,7 +283,7 @@ def run(
duration: Video duration filter.
license_videos: License filter.
max_results: Maximum number of results.
-
+
Returns:
List of video result dictionaries.
"""
diff --git a/webscout/search/engines/yahoo/weather.py b/webscout/search/engines/yahoo/weather.py
index 9044b9e4..c9ae4dcb 100644
--- a/webscout/search/engines/yahoo/weather.py
+++ b/webscout/search/engines/yahoo/weather.py
@@ -3,7 +3,6 @@
from __future__ import annotations
import re
-import json
from typing import Any
from ...http_client import HttpClient
@@ -14,7 +13,7 @@ class YahooWeather:
def __init__(self, proxy: str | None = None, timeout: int | None = None, verify: bool = True):
"""Initialize weather search engine.
-
+
Args:
proxy: Proxy URL.
timeout: Request timeout in seconds.
@@ -32,22 +31,22 @@ def request(self, method: str, url: str, **kwargs: Any) -> str | None:
def run(self, *args, **kwargs) -> list[dict[str, Any]]:
"""Get weather data from Yahoo.
-
+
Args:
location: Location to get weather for (e.g., "New York", "London", "Bengaluru")
-
+
Returns:
List of weather data dictionaries
"""
location = args[0] if args else kwargs.get("location") or kwargs.get("keywords")
-
+
if not location:
raise ValueError("Location is required for weather search")
-
+
try:
# Use the search endpoint which redirects to the correct weather page
search_url = f"https://weather.yahoo.com/search/?q={location.replace(' ', '+')}"
-
+
# Fetch the page
response = self.request("GET", search_url)
if not response:
@@ -55,25 +54,25 @@ def run(self, *args, **kwargs) -> list[dict[str, Any]]:
"location": location,
"error": "Failed to fetch weather data from Yahoo"
}]
-
+
# Extract JSON data from the page
weather_data = self._extract_json_data(response, location)
-
+
if weather_data:
return [weather_data]
-
+
# Fallback: try regex parsing
return self._parse_weather_html(response, location)
-
+
except Exception as e:
return [{
"location": location,
"error": f"Failed to fetch weather data: {str(e)}"
}]
-
+
def _extract_json_data(self, html: str, location: str) -> dict[str, Any] | None:
"""Extract weather data from embedded JSON in the page.
-
+
Yahoo Weather embeds JSON data in script tags that can be parsed.
"""
try:
@@ -81,53 +80,53 @@ def _extract_json_data(self, html: str, location: str) -> dict[str, Any] | None:
# Pattern: self.__next_f.push([1,"..JSON data.."])
json_pattern = r'self\.__next_f\.push\(\[1,"([^"]+)"\]\)'
matches = re.findall(json_pattern, html)
-
+
weather_info = {}
-
+
for match in matches:
# Unescape the JSON string
try:
# The data is escaped, so we need to decode it
decoded = match.encode().decode('unicode_escape')
-
+
# Look for temperature data
temp_match = re.search(r'"temperature":(\d+)', decoded)
if temp_match and not weather_info.get('temperature'):
weather_info['temperature'] = int(temp_match.group(1))
-
+
# Look for condition
condition_match = re.search(r'"iconLabel":"([^"]+)"', decoded)
if condition_match and not weather_info.get('condition'):
weather_info['condition'] = condition_match.group(1)
-
+
# Look for high/low
high_match = re.search(r'"highTemperature":(\d+)', decoded)
if high_match and not weather_info.get('high'):
weather_info['high'] = int(high_match.group(1))
-
+
low_match = re.search(r'"lowTemperature":(\d+)', decoded)
if low_match and not weather_info.get('low'):
weather_info['low'] = int(low_match.group(1))
-
+
# Look for humidity
humidity_match = re.search(r'"value":"(\d+)%"[^}]*"category":"Humidity"', decoded)
if humidity_match and not weather_info.get('humidity'):
weather_info['humidity'] = int(humidity_match.group(1))
-
+
# Look for precipitation probability
precip_match = re.search(r'"probabilityOfPrecipitation":"(\d+)%"', decoded)
if precip_match and not weather_info.get('precipitation_chance'):
weather_info['precipitation_chance'] = int(precip_match.group(1))
-
+
# Look for location name
location_match = re.search(r'"name":"([^"]+)","code":null,"woeid":(\d+)', decoded)
if location_match and not weather_info.get('location_name'):
weather_info['location_name'] = location_match.group(1)
weather_info['woeid'] = int(location_match.group(2))
-
+
except Exception:
continue
-
+
if weather_info and weather_info.get('temperature'):
return {
"location": weather_info.get('location_name', location),
@@ -141,78 +140,78 @@ def _extract_json_data(self, html: str, location: str) -> dict[str, Any] | None:
"source": "Yahoo Weather",
"units": "Fahrenheit"
}
-
+
return None
-
- except Exception as e:
+
+ except Exception:
return None
-
+
def _parse_weather_html(self, html_content: str, location: str) -> list[dict[str, Any]]:
"""Fallback: Parse weather data from HTML content using regex.
-
+
Args:
html_content: HTML content of weather page
location: Location name
-
+
Returns:
List of weather data dictionaries
"""
try:
weather_data = {"location": location}
-
+
# Extract current temperature
temp_patterns = [
r'
]*class="[^"]*font-title1[^"]*"[^>]*>(\d+)°
',
r'>(\d+)°<',
r'"temperature":(\d+)',
]
-
+
for pattern in temp_patterns:
match = re.search(pattern, html_content)
if match:
weather_data["temperature_f"] = int(match.group(1))
break
-
+
# Extract condition
condition_patterns = [
r'"iconLabel":"([^"]+)"',
r'aria-label="([^"]*(?:Cloudy|Sunny|Rain|Clear|Thunder|Shower|Fog)[^"]*)"',
]
-
+
for pattern in condition_patterns:
match = re.search(pattern, html_content, re.IGNORECASE)
if match:
weather_data["condition"] = match.group(1)
break
-
+
# Extract high/low
high_match = re.search(r'"highTemperature":(\d+)', html_content)
if high_match:
weather_data["high_f"] = int(high_match.group(1))
-
+
low_match = re.search(r'"lowTemperature":(\d+)', html_content)
if low_match:
weather_data["low_f"] = int(low_match.group(1))
-
+
# Extract humidity
humidity_match = re.search(r'Humidity[^>]*>(\d+)%|"value":"(\d+)%"[^}]*"Humidity"', html_content, re.IGNORECASE)
if humidity_match:
weather_data["humidity_percent"] = int(humidity_match.group(1) or humidity_match.group(2))
-
+
weather_data["source"] = "Yahoo Weather"
weather_data["units"] = "Fahrenheit"
-
+
# Remove None values
weather_data = {k: v for k, v in weather_data.items() if v is not None}
-
+
if len(weather_data) > 3: # Has more than just location, source, and units
return [weather_data]
-
+
return [{
"location": location,
"error": "Could not extract weather data from page"
}]
-
+
except Exception as e:
return [{
"location": location,
diff --git a/webscout/search/engines/yandex.py b/webscout/search/engines/yandex.py
index 59b095d9..d79300c6 100644
--- a/webscout/search/engines/yandex.py
+++ b/webscout/search/engines/yandex.py
@@ -44,13 +44,13 @@ def build_payload(
def run(self, *args, **kwargs) -> list[TextResult]:
"""Run text search on Yandex.
-
+
Args:
keywords: Search query.
region: Region code.
safesearch: Safe search level.
max_results: Maximum number of results (ignored for now).
-
+
Returns:
List of TextResult objects.
"""
@@ -58,7 +58,7 @@ def run(self, *args, **kwargs) -> list[TextResult]:
region = args[1] if len(args) > 1 else kwargs.get("region", "us-en")
safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
max_results = args[3] if len(args) > 3 else kwargs.get("max_results")
-
+
results = self.search(query=keywords, region=region, safesearch=safesearch)
if results and max_results:
results = results[:max_results]
diff --git a/webscout/search/engines/yep/base.py b/webscout/search/engines/yep/base.py
index ff3dac76..297d9ccc 100644
--- a/webscout/search/engines/yep/base.py
+++ b/webscout/search/engines/yep/base.py
@@ -1,8 +1,9 @@
from __future__ import annotations
-from ....litagent import LitAgent
from curl_cffi.requests import Session
+from ....litagent import LitAgent
+
class YepBase:
"""Base class for Yep search engines."""
diff --git a/webscout/search/engines/yep/images.py b/webscout/search/engines/yep/images.py
index aa6fedfc..131762f8 100644
--- a/webscout/search/engines/yep/images.py
+++ b/webscout/search/engines/yep/images.py
@@ -1,11 +1,12 @@
from __future__ import annotations
-from typing import List, Optional
+from typing import List
from urllib.parse import urlencode
-from .base import YepBase
from webscout.search.results import ImagesResult
+from .base import YepBase
+
class YepImages(YepBase):
name = "yep"
diff --git a/webscout/search/engines/yep/text.py b/webscout/search/engines/yep/text.py
index 6b97d794..a739ec12 100644
--- a/webscout/search/engines/yep/text.py
+++ b/webscout/search/engines/yep/text.py
@@ -1,11 +1,12 @@
from __future__ import annotations
-from typing import List, Optional
+from typing import List
from urllib.parse import urlencode
-from .base import YepBase
from webscout.search.results import TextResult
+from .base import YepBase
+
class YepSearch(YepBase):
name = "yep"
diff --git a/webscout/search/http_client.py b/webscout/search/http_client.py
index 8f68631f..fc8f4da0 100644
--- a/webscout/search/http_client.py
+++ b/webscout/search/http_client.py
@@ -2,7 +2,6 @@
from __future__ import annotations
-from litprinter import ic
from random import choice
from typing import Any, Literal
@@ -18,7 +17,7 @@
class HttpClient:
"""HTTP client wrapper for search engines."""
-
+
# curl_cffi supported browser impersonations
_impersonates = (
"chrome99", "chrome100", "chrome101", "chrome104", "chrome107", "chrome110",
@@ -37,7 +36,7 @@ def __init__(
headers: dict[str, str] | None = None,
) -> None:
"""Initialize HTTP client.
-
+
Args:
proxy: Proxy URL (supports http/https/socks5).
timeout: Request timeout in seconds.
@@ -47,10 +46,10 @@ def __init__(
self.proxy = proxy
self.timeout = timeout
self.verify = verify
-
+
# Choose random browser to impersonate
impersonate_browser = choice(self._impersonates)
-
+
# Initialize curl_cffi session
self.client = curl_cffi.requests.Session(
headers=headers or {},
@@ -59,7 +58,7 @@ def __init__(
impersonate=impersonate_browser,
verify=verify,
)
-
+
def request(
self,
method: Literal["GET", "POST", "HEAD", "OPTIONS", "DELETE", "PUT", "PATCH"],
@@ -73,7 +72,7 @@ def request(
**kwargs: Any,
) -> curl_cffi.requests.Response:
"""Make HTTP request.
-
+
Args:
method: HTTP method.
url: Request URL.
@@ -84,10 +83,10 @@ def request(
cookies: Request cookies.
timeout: Request timeout (overrides default).
**kwargs: Additional arguments passed to curl_cffi.
-
+
Returns:
Response object.
-
+
Raises:
TimeoutE: Request timeout.
RatelimitE: Rate limit exceeded.
@@ -101,15 +100,15 @@ def request(
"timeout": timeout or self.timeout,
**kwargs,
}
-
+
if isinstance(cookies, dict):
request_kwargs["cookies"] = cookies
-
+
if data is not None:
request_kwargs["data"] = data
-
+
resp = self.client.request(method, url, **request_kwargs)
-
+
# Check response status
if resp.status_code == 200:
return resp
@@ -117,38 +116,38 @@ def request(
raise RatelimitE(f"{resp.url} {resp.status_code} Rate limit")
else:
raise WebscoutE(f"{resp.url} returned {resp.status_code}")
-
+
except Exception as ex:
if "time" in str(ex).lower() or "timeout" in str(ex).lower():
raise TimeoutE(f"{url} {type(ex).__name__}: {ex}") from ex
raise WebscoutE(f"{url} {type(ex).__name__}: {ex}") from ex
-
+
def get(self, url: str, **kwargs: Any) -> curl_cffi.requests.Response:
"""Make GET request."""
return self.request("GET", url, **kwargs)
-
+
def post(self, url: str, **kwargs: Any) -> curl_cffi.requests.Response:
"""Make POST request."""
return self.request("POST", url, **kwargs)
-
+
def set_cookies(self, url: str, cookies: dict[str, str]) -> None:
"""Set cookies for a domain.
-
+
Args:
url: URL to set cookies for.
cookies: Cookie dictionary.
"""
self.client.cookies.update(cookies)
-
+
def close(self) -> None:
"""Close the HTTP client."""
if hasattr(self.client, 'close'):
self.client.close()
-
+
def __enter__(self) -> HttpClient:
"""Context manager entry."""
return self
-
+
def __exit__(self, *args: Any) -> None:
"""Context manager exit."""
self.close()
diff --git a/webscout/search/results.py b/webscout/search/results.py
index e9d0cc7e..a740f603 100644
--- a/webscout/search/results.py
+++ b/webscout/search/results.py
@@ -9,11 +9,11 @@
@dataclass
class TextResult:
"""Text search result."""
-
+
title: str = ""
href: str = ""
body: str = ""
-
+
def to_dict(self) -> dict[str, Any]:
"""Convert to dictionary."""
return {
@@ -26,7 +26,7 @@ def to_dict(self) -> dict[str, Any]:
@dataclass
class ImagesResult:
"""Images search result."""
-
+
title: str = ""
image: str = ""
thumbnail: str = ""
@@ -34,7 +34,7 @@ class ImagesResult:
height: int = 0
width: int = 0
source: str = ""
-
+
def to_dict(self) -> dict[str, Any]:
"""Convert to dictionary."""
return {
@@ -51,7 +51,7 @@ def to_dict(self) -> dict[str, Any]:
@dataclass
class VideosResult:
"""Videos search result."""
-
+
content: str = ""
description: str = ""
duration: str = ""
@@ -65,7 +65,7 @@ class VideosResult:
statistics: dict[str, int] = field(default_factory=dict)
title: str = ""
uploader: str = ""
-
+
def to_dict(self) -> dict[str, Any]:
"""Convert to dictionary."""
return {
@@ -88,14 +88,14 @@ def to_dict(self) -> dict[str, Any]:
@dataclass
class NewsResult:
"""News search result."""
-
+
date: str = ""
title: str = ""
body: str = ""
url: str = ""
image: str = ""
source: str = ""
-
+
def to_dict(self) -> dict[str, Any]:
"""Convert to dictionary."""
return {
@@ -111,7 +111,7 @@ def to_dict(self) -> dict[str, Any]:
@dataclass
class BooksResult:
"""Books search result."""
-
+
title: str = ""
author: str = ""
href: str = ""
@@ -121,7 +121,7 @@ class BooksResult:
language: str = ""
filesize: str = ""
extension: str = ""
-
+
def to_dict(self) -> dict[str, Any]:
"""Convert to dictionary."""
return {
diff --git a/webscout/search/yahoo_main.py b/webscout/search/yahoo_main.py
index d7f1bdf1..777a7e28 100644
--- a/webscout/search/yahoo_main.py
+++ b/webscout/search/yahoo_main.py
@@ -1,19 +1,21 @@
"""Yahoo unified search interface."""
from __future__ import annotations
-from typing import List, Optional
+
+from typing import Dict, List, Optional
+
from .base import BaseSearch
-from .engines.yahoo.text import YahooText
+from .engines.yahoo.answers import YahooAnswers
from .engines.yahoo.images import YahooImages
-from .engines.yahoo.videos import YahooVideos
+from .engines.yahoo.maps import YahooMaps
from .engines.yahoo.news import YahooNews
from .engines.yahoo.suggestions import YahooSuggestions
-from .engines.yahoo.answers import YahooAnswers
-from .engines.yahoo.maps import YahooMaps
+from .engines.yahoo.text import YahooText
from .engines.yahoo.translate import YahooTranslate
+from .engines.yahoo.videos import YahooVideos
from .engines.yahoo.weather import YahooWeather
-from .results import TextResult, ImagesResult, VideosResult, NewsResult
-from typing import Dict
+from .results import ImagesResult, NewsResult, TextResult, VideosResult
+
class YahooSearch(BaseSearch):
"""Unified Yahoo search interface."""
@@ -52,4 +54,4 @@ def translate(self, keywords: str, from_lang: Optional[str] = None, to_lang: str
def weather(self, keywords: str) -> List[Dict[str, str]]:
search = YahooWeather()
- return search.run(keywords)
\ No newline at end of file
+ return search.run(keywords)
diff --git a/webscout/search/yep_main.py b/webscout/search/yep_main.py
index 7ce1837c..e8234cd1 100644
--- a/webscout/search/yep_main.py
+++ b/webscout/search/yep_main.py
@@ -1,11 +1,13 @@
"""Yep unified search interface."""
from __future__ import annotations
+
from typing import Dict, List, Optional
+
from .base import BaseSearch
-from .engines.yep.text import YepSearch as YepTextSearch
from .engines.yep.images import YepImages
from .engines.yep.suggestions import YepSuggestions
+from .engines.yep.text import YepSearch as YepTextSearch
class YepSearch(BaseSearch):
diff --git a/webscout/server/__init__.py b/webscout/server/__init__.py
index 51fa9f14..bc912bd7 100644
--- a/webscout/server/__init__.py
+++ b/webscout/server/__init__.py
@@ -1,4 +1,7 @@
# webscout/server/__init__.py
+from .exceptions import APIError
+from .routes import Api
+
# Import server functions lazily to avoid module execution issues
def create_app():
@@ -12,8 +15,8 @@ def run_api(*args, **kwargs):
def start_server(*args, **kwargs):
from .server import start_server as _start_server
return _start_server(*args, **kwargs)
-from .routes import Api
-from .exceptions import APIError
+
+
# Lazy imports for config classes to avoid initialization issues
def get_server_config():
@@ -42,4 +45,4 @@ def initialize_tti_provider_map():
"APIError",
"initialize_provider_map",
"initialize_tti_provider_map"
-]
\ No newline at end of file
+]
diff --git a/webscout/server/config.py b/webscout/server/config.py
index 5e0faaa2..61f37f62 100644
--- a/webscout/server/config.py
+++ b/webscout/server/config.py
@@ -3,7 +3,8 @@
"""
import os
-from typing import List, Dict, Optional, Any
+from typing import Any, Dict, List, Optional
+
from litprinter import ic
# Configuration constants
@@ -33,7 +34,8 @@ def update(self, **kwargs) -> None:
for key, value in kwargs.items():
if hasattr(self, key) and value is not None:
setattr(self, key, value)
- ic.configureOutput(prefix='INFO| '); ic(f"Config updated: {key} = {value}")
+ ic.configureOutput(prefix='INFO| ')
+ ic(f"Config updated: {key} = {value}")
def validate(self) -> None:
"""Validate configuration settings."""
@@ -42,7 +44,8 @@ def validate(self) -> None:
if self.default_provider not in self.provider_map and self.provider_map:
available_providers = list(set(v.__name__ for v in self.provider_map.values()))
- ic.configureOutput(prefix='WARNING| '); ic(f"Default provider '{self.default_provider}' not found. Available: {available_providers}")
+ ic.configureOutput(prefix='WARNING| ')
+ ic(f"Default provider '{self.default_provider}' not found. Available: {available_providers}")
class AppConfig:
@@ -72,4 +75,4 @@ def set_config(cls, **data):
config.update(**filtered_data)
except ImportError:
# Handle case where server module is not available
- pass
\ No newline at end of file
+ pass
diff --git a/webscout/server/exceptions.py b/webscout/server/exceptions.py
index be3bb106..00b2cb8e 100644
--- a/webscout/server/exceptions.py
+++ b/webscout/server/exceptions.py
@@ -5,8 +5,10 @@
import json
import re
from typing import Optional
+
from fastapi.responses import JSONResponse
from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR
+
from .request_models import ErrorDetail, ErrorResponse
@@ -14,10 +16,10 @@ def clean_text(text):
"""Clean text by removing null bytes and control characters except newlines and tabs."""
if not isinstance(text, str):
return text
-
+
# Remove null bytes
text = text.replace('\x00', '')
-
+
# Keep newlines, tabs, and other printable characters, remove other control chars
# This regex matches control characters except \n, \r, \t
return re.sub(r'[\x01-\x08\x0b\x0c\x0e-\x1f\x7f]', '', text)
diff --git a/webscout/server/providers.py b/webscout/server/providers.py
index 45dee819..c9f1478e 100644
--- a/webscout/server/providers.py
+++ b/webscout/server/providers.py
@@ -2,11 +2,12 @@
Provider management and initialization for the Webscout API.
"""
-import sys
import inspect
+import sys
from typing import Any, Dict, Tuple
-from starlette.status import HTTP_404_NOT_FOUND, HTTP_500_INTERNAL_SERVER_ERROR
+
from litprinter import ic
+from starlette.status import HTTP_404_NOT_FOUND, HTTP_500_INTERNAL_SERVER_ERROR
from .config import AppConfig
from .exceptions import APIError
@@ -18,7 +19,8 @@
def initialize_provider_map() -> None:
"""Initialize the provider map by discovering available providers."""
- ic.configureOutput(prefix='INFO| '); ic("Initializing provider map...")
+ ic.configureOutput(prefix='INFO| ')
+ ic("Initializing provider map...")
try:
from webscout.Provider.OPENAI.base import OpenAICompatibleProvider
@@ -34,7 +36,7 @@ def initialize_provider_map() -> None:
and obj.__name__ != "OpenAICompatibleProvider"
):
# Only include providers that don't require authentication
- if hasattr(obj, 'required_auth') and getattr(obj, 'required_auth', True) == False:
+ if hasattr(obj, 'required_auth') and not getattr(obj, 'required_auth', True):
provider_name = obj.__name__
AppConfig.provider_map[provider_name] = obj
provider_count += 1
@@ -51,7 +53,8 @@ def initialize_provider_map() -> None:
# Fallback to ChatGPT if no providers found
if not AppConfig.provider_map:
- ic.configureOutput(prefix='WARNING| '); ic("No providers found, using ChatGPT fallback")
+ ic.configureOutput(prefix='WARNING| ')
+ ic("No providers found, using ChatGPT fallback")
try:
from webscout.Provider.OPENAI.chatgpt import ChatGPT
fallback_models = ["gpt-4", "gpt-4o", "gpt-4o-mini", "gpt-3.5-turbo"]
@@ -66,19 +69,23 @@ def initialize_provider_map() -> None:
provider_count = 1
model_count = len(fallback_models)
except ImportError as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Failed to import ChatGPT fallback: {e}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Failed to import ChatGPT fallback: {e}")
raise APIError("No providers available", HTTP_500_INTERNAL_SERVER_ERROR)
- ic.configureOutput(prefix='INFO| '); ic(f"Initialized {provider_count} providers with {model_count} models")
+ ic.configureOutput(prefix='INFO| ')
+ ic(f"Initialized {provider_count} providers with {model_count} models")
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Failed to initialize provider map: {e}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Failed to initialize provider map: {e}")
raise APIError(f"Provider initialization failed: {e}", HTTP_500_INTERNAL_SERVER_ERROR)
def initialize_tti_provider_map() -> None:
"""Initialize the TTI provider map by discovering available TTI providers."""
- ic.configureOutput(prefix='INFO| '); ic("Initializing TTI provider map...")
+ ic.configureOutput(prefix='INFO| ')
+ ic("Initializing TTI provider map...")
try:
from webscout.Provider.TTI.base import TTICompatibleProvider
@@ -110,7 +117,8 @@ def initialize_tti_provider_map() -> None:
# Fallback to PollinationsAI if no TTI providers found
if not AppConfig.tti_provider_map:
- ic.configureOutput(prefix='WARNING| '); ic("No TTI providers found, using PollinationsAI fallback")
+ ic.configureOutput(prefix='WARNING| ')
+ ic("No TTI providers found, using PollinationsAI fallback")
try:
from webscout.Provider.TTI.pollinations import PollinationsAI
fallback_models = ["flux", "turbo", "gptimage"]
@@ -125,13 +133,16 @@ def initialize_tti_provider_map() -> None:
provider_count = 1
model_count = len(fallback_models)
except ImportError as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Failed to import PollinationsAI fallback: {e}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Failed to import PollinationsAI fallback: {e}")
raise APIError("No TTI providers available", HTTP_500_INTERNAL_SERVER_ERROR)
- ic.configureOutput(prefix='INFO| '); ic(f"Initialized {provider_count} TTI providers with {model_count} models")
+ ic.configureOutput(prefix='INFO| ')
+ ic(f"Initialized {provider_count} TTI providers with {model_count} models")
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Failed to initialize TTI provider map: {e}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Failed to initialize TTI provider map: {e}")
raise APIError(f"TTI Provider initialization failed: {e}", HTTP_500_INTERNAL_SERVER_ERROR)
diff --git a/webscout/server/request_models.py b/webscout/server/request_models.py
index 2bdd8fa2..284f374d 100644
--- a/webscout/server/request_models.py
+++ b/webscout/server/request_models.py
@@ -2,7 +2,8 @@
Pydantic models for API requests and responses.
"""
-from typing import List, Dict, Optional, Union, Literal
+from typing import Dict, List, Literal, Optional, Union
+
from webscout.Provider.OPENAI.pydantic_imports import BaseModel, Field
diff --git a/webscout/server/request_processing.py b/webscout/server/request_processing.py
index 9017ffca..987b73b9 100644
--- a/webscout/server/request_processing.py
+++ b/webscout/server/request_processing.py
@@ -5,18 +5,23 @@
import json
import time
import uuid
-from typing import List, Dict, Any
-from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY, HTTP_500_INTERNAL_SERVER_ERROR
-from fastapi.responses import StreamingResponse
+from typing import Any, Dict, List
-from webscout.Provider.OPENAI.utils import ChatCompletion, Choice, ChatCompletionMessage, CompletionUsage
+from fastapi.responses import StreamingResponse
from litprinter import ic
+from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY, HTTP_500_INTERNAL_SERVER_ERROR
-from .request_models import Message, ChatCompletionRequest
-from .exceptions import APIError, clean_text
+from webscout.Provider.OPENAI.utils import (
+ ChatCompletion,
+ ChatCompletionMessage,
+ Choice,
+ CompletionUsage,
+)
# from .simple_logger import log_api_request, get_client_ip, generate_request_id
from .config import AppConfig
+from .exceptions import APIError, clean_text
+from .request_models import ChatCompletionRequest, Message
def get_client_ip(request) -> str:
@@ -24,11 +29,11 @@ def get_client_ip(request) -> str:
forwarded_for = request.headers.get("X-Forwarded-For")
if forwarded_for:
return forwarded_for.split(",")[0].strip()
-
+
real_ip = request.headers.get("X-Real-IP")
if real_ip:
return real_ip.strip()
-
+
return getattr(request.client, "host", "unknown")
@@ -60,7 +65,7 @@ async def log_request(request_id: str, ip_address: str, model_used: str, questio
user_agent = None
if request_obj:
user_agent = request_obj.headers.get("user-agent")
-
+
await log_api_request(
request_id=request_id,
ip_address=ip_address,
@@ -73,7 +78,8 @@ async def log_request(request_id: str, ip_address: str, model_used: str, questio
user_agent=user_agent
)
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Failed to log request {request_id}: {e}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Failed to log request {request_id}: {e}")
# Don't raise exception to avoid breaking the main request flow
@@ -134,15 +140,16 @@ def prepare_provider_params(chat_request: ChatCompletionRequest, model_name: str
async def handle_streaming_response(provider: Any, params: Dict[str, Any], request_id: str,
- ip_address: str, question: str, model_name: str, start_time: float,
+ ip_address: str, question: str, model_name: str, start_time: float,
provider_name: str = None, request_obj=None) -> StreamingResponse:
"""Handle streaming chat completion response."""
collected_content = []
-
+
async def streaming():
nonlocal collected_content
try:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Starting streaming response for request {request_id}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Starting streaming response for request {request_id}")
completion_stream = provider.chat.completions.create(**params)
# Check if it's iterable (generator, iterator, or other iterable types)
@@ -158,7 +165,7 @@ async def streaming():
chunk_data = chunk
else: # Fallback for unknown chunk types
chunk_data = chunk
-
+
# Clean text content in the chunk to remove control characters
if isinstance(chunk_data, dict) and 'choices' in chunk_data:
for choice in chunk_data.get('choices', []):
@@ -175,10 +182,11 @@ async def streaming():
if content:
collected_content.append(content)
choice['message']['content'] = clean_text(content)
-
+
yield f"data: {json.dumps(chunk_data, ensure_ascii=False)}\n\n"
except TypeError as te:
- ic.configureOutput(prefix='ERROR| '); ic(f"Error iterating over completion_stream: {te}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Error iterating over completion_stream: {te}")
# Fall back to treating as non-generator response
if hasattr(completion_stream, 'model_dump'):
response_data = completion_stream.model_dump(exclude_none=True)
@@ -186,7 +194,7 @@ async def streaming():
response_data = completion_stream.dict(exclude_none=True)
else:
response_data = completion_stream
-
+
# Clean text content in the response
if isinstance(response_data, dict) and 'choices' in response_data:
for choice in response_data.get('choices', []):
@@ -201,7 +209,7 @@ async def streaming():
if content:
collected_content.append(content)
choice['message']['content'] = clean_text(content)
-
+
yield f"data: {json.dumps(response_data, ensure_ascii=False)}\n\n"
else: # Non-generator response
if hasattr(completion_stream, 'model_dump'):
@@ -210,7 +218,7 @@ async def streaming():
response_data = completion_stream.dict(exclude_none=True)
else:
response_data = completion_stream
-
+
# Clean text content in the response
if isinstance(response_data, dict) and 'choices' in response_data:
for choice in response_data.get('choices', []):
@@ -225,11 +233,12 @@ async def streaming():
if content:
collected_content.append(content)
choice['message']['content'] = clean_text(content)
-
+
yield f"data: {json.dumps(response_data, ensure_ascii=False)}\n\n"
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Error in streaming response for request {request_id}: {e}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Error in streaming response for request {request_id}: {e}")
error_message = clean_text(str(e))
error_data = {
"error": {
@@ -239,7 +248,7 @@ async def streaming():
}
}
yield f"data: {json.dumps(error_data, ensure_ascii=False)}\n\n"
-
+
# Log error request
response_time_ms = int((time.time() - start_time) * 1000)
await log_request(
@@ -256,7 +265,7 @@ async def streaming():
)
finally:
yield "data: [DONE]\n\n"
-
+
# Log successful streaming request
if collected_content:
answer = "".join(collected_content)
@@ -272,17 +281,18 @@ async def streaming():
provider=provider_name,
request_obj=request_obj
)
-
+
return StreamingResponse(streaming(), media_type="text/event-stream")
async def handle_non_streaming_response(provider: Any, params: Dict[str, Any],
request_id: str, start_time: float, ip_address: str,
- question: str, model_name: str, provider_name: str = None,
+ question: str, model_name: str, provider_name: str = None,
request_obj=None) -> Dict[str, Any]:
"""Handle non-streaming chat completion response."""
try:
- ic.configureOutput(prefix='DEBUG| '); ic(f"Starting non-streaming response for request {request_id}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Starting non-streaming response for request {request_id}")
completion = provider.chat.completions.create(**params)
if completion is None:
@@ -298,7 +308,7 @@ async def handle_non_streaming_response(provider: Any, params: Dict[str, Any],
)],
usage=CompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
).model_dump(exclude_none=True)
-
+
# Log error request
response_time_ms = int((time.time() - start_time) * 1000)
await log_request(
@@ -313,7 +323,7 @@ async def handle_non_streaming_response(provider: Any, params: Dict[str, Any],
provider=provider_name,
request_obj=request_obj
)
-
+
return error_response
# Standardize response format
@@ -329,7 +339,7 @@ async def handle_non_streaming_response(provider: Any, params: Dict[str, Any],
HTTP_500_INTERNAL_SERVER_ERROR,
"provider_error"
)
-
+
# Extract answer from response and clean text content
answer = ""
if isinstance(response_data, dict) and 'choices' in response_data:
@@ -343,7 +353,8 @@ async def handle_non_streaming_response(provider: Any, params: Dict[str, Any],
elapsed = time.time() - start_time
response_time_ms = int(elapsed * 1000)
- ic.configureOutput(prefix='INFO| '); ic(f"Completed non-streaming request {request_id} in {elapsed:.2f}s")
+ ic.configureOutput(prefix='INFO| ')
+ ic(f"Completed non-streaming request {request_id} in {elapsed:.2f}s")
# Log successful request
await log_request(
@@ -361,9 +372,10 @@ async def handle_non_streaming_response(provider: Any, params: Dict[str, Any],
return response_data
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Error in non-streaming response for request {request_id}: {e}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Error in non-streaming response for request {request_id}: {e}")
error_message = clean_text(str(e))
-
+
# Log error request
response_time_ms = int((time.time() - start_time) * 1000)
await log_request(
@@ -378,7 +390,7 @@ async def handle_non_streaming_response(provider: Any, params: Dict[str, Any],
provider=provider_name,
request_obj=request_obj
)
-
+
raise APIError(
f"Provider error: {error_message}",
HTTP_500_INTERNAL_SERVER_ERROR,
diff --git a/webscout/server/routes.py b/webscout/server/routes.py
index 0870d9cf..2a0a9dc6 100644
--- a/webscout/server/routes.py
+++ b/webscout/server/routes.py
@@ -5,49 +5,52 @@
import time
import uuid
-from fastapi import FastAPI, Request, Body, Query
-from fastapi.responses import JSONResponse
+from fastapi import Body, FastAPI, Query, Request
from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from litprinter import ic
from starlette.exceptions import HTTPException as StarletteHTTPException
from starlette.status import (
HTTP_500_INTERNAL_SERVER_ERROR,
)
-from litprinter import ic
+from webscout.search.engines import ENGINES
from .config import AppConfig
-from .request_models import (
- ChatCompletionRequest, ImageGenerationRequest, ModelListResponse
-)
from .exceptions import APIError
from .providers import (
- resolve_provider_and_model, resolve_tti_provider_and_model,
- get_provider_instance, get_tti_provider_instance
+ get_provider_instance,
+ get_tti_provider_instance,
+ resolve_provider_and_model,
+ resolve_tti_provider_and_model,
)
+from .request_models import ChatCompletionRequest, ImageGenerationRequest, ModelListResponse
from .request_processing import (
- process_messages, prepare_provider_params,
- handle_streaming_response, handle_non_streaming_response
+ handle_non_streaming_response,
+ handle_streaming_response,
+ prepare_provider_params,
+ process_messages,
)
-from webscout.search.engines import ENGINES
+
class Api:
"""API route handler class."""
-
+
def __init__(self, app: FastAPI) -> None:
self.app = app
def register_validation_exception_handler(self):
"""Register comprehensive exception handlers."""
- from fastapi.exceptions import RequestValidationError
- from starlette.exceptions import HTTPException as StarletteHTTPException
from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY, HTTP_500_INTERNAL_SERVER_ERROR
+
from .exceptions import APIError
-
+
github_footer = "If you believe this is a bug, please pull an issue at https://github.com/OEvortex/Webscout."
@self.app.exception_handler(APIError)
async def api_error_handler(request, exc: APIError):
- ic.configureOutput(prefix='ERROR| '); ic(f"API Error: {exc.message} (Status: {exc.status_code})")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"API Error: {exc.message} (Status: {exc.status_code})")
# Patch: add footer to error content before creating JSONResponse
error_response = exc.to_response()
# If the response is a JSONResponse, patch its content dict before returning
@@ -68,7 +71,7 @@ async def validation_exception_handler(request, exc: RequestValidationError):
errors = exc.errors()
error_messages = []
body = await request.body()
- is_empty_body = not body or body.strip() in (b"", b"null", b"{}")
+ not body or body.strip() in (b"", b"null", b"{}")
for error in errors:
loc = error.get("loc", [])
loc_str = " -> ".join(str(item) for item in loc)
@@ -101,7 +104,8 @@ async def http_exception_handler(request, exc: StarletteHTTPException):
@self.app.exception_handler(Exception)
async def general_exception_handler(request, exc: Exception):
- ic.configureOutput(prefix='ERROR| '); ic(f"Unhandled server error: {exc}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Unhandled server error: {exc}")
content = {
"error": {
"message": f"Internal server error: {str(exc)}",
@@ -126,7 +130,7 @@ async def health_check():
return {"status": "healthy", "service": "webscout-api", "version": "0.2.0"}
def _register_model_routes(self):
- """Register model listing routes."""
+ """Register model listing routes."""
@self.app.get(
"/v1/models",
response_model=ModelListResponse,
@@ -183,32 +187,32 @@ async def retrieve_model(model: str):
async def list_providers():
"""Get information about all available chat completion providers."""
providers = {}
-
+
# Extract unique provider names (exclude model mappings)
provider_names = set()
for key, provider_class in AppConfig.provider_map.items():
if "/" not in key: # Provider name, not model mapping
provider_names.add(key)
-
+
for provider_name in sorted(provider_names):
provider_class = AppConfig.provider_map[provider_name]
-
+
# Get available models for this provider
models = []
for key, cls in AppConfig.provider_map.items():
if key.startswith(f"{provider_name}/"):
model_name = key.split("/", 1)[1]
models.append(model_name)
-
+
# Sort models
models = sorted(models)
-
+
# Get supported parameters (common OpenAI-compatible parameters)
supported_params = [
- "model", "messages", "max_tokens", "temperature", "top_p",
+ "model", "messages", "max_tokens", "temperature", "top_p",
"presence_penalty", "frequency_penalty", "stop", "stream", "user"
]
-
+
providers[provider_name] = {
"name": provider_name,
"class": provider_class.__name__,
@@ -216,12 +220,12 @@ async def list_providers():
"parameters": supported_params,
"model_count": len(models)
}
-
+
return {
"providers": providers,
"total_providers": len(providers)
}
-
+
@self.app.get(
"/v1/TTI/models",
response_model=ModelListResponse,
@@ -256,32 +260,32 @@ async def list_tti_models():
async def list_tti_providers():
"""Get information about all available TTI providers."""
providers = {}
-
+
# Extract unique provider names (exclude model mappings)
provider_names = set()
for key, provider_class in AppConfig.tti_provider_map.items():
if "/" not in key: # Provider name, not model mapping
provider_names.add(key)
-
+
for provider_name in sorted(provider_names):
provider_class = AppConfig.tti_provider_map[provider_name]
-
+
# Get available models for this provider
models = []
for key, cls in AppConfig.tti_provider_map.items():
if key.startswith(f"{provider_name}/"):
model_name = key.split("/", 1)[1]
models.append(model_name)
-
+
# Sort models
models = sorted(models)
-
+
# Get supported parameters (common TTI parameters)
supported_params = [
- "prompt", "model", "n", "size", "response_format", "user",
+ "prompt", "model", "n", "size", "response_format", "user",
"style", "aspect_ratio", "timeout", "image_format", "seed"
]
-
+
providers[provider_name] = {
"name": provider_name,
"class": provider_class.__name__,
@@ -289,7 +293,7 @@ async def list_tti_providers():
"parameters": supported_params,
"model_count": len(models)
}
-
+
return {
"providers": providers,
"total_providers": len(providers)
@@ -325,7 +329,8 @@ async def chat_completions(
request_id = f"chatcmpl-{uuid.uuid4()}"
try:
- ic.configureOutput(prefix='INFO| '); ic(f"Processing chat completion request {request_id} for model: {chat_request.model}")
+ ic.configureOutput(prefix='INFO| ')
+ ic(f"Processing chat completion request {request_id} for model: {chat_request.model}")
# Resolve provider and model
provider_class, model_name = resolve_provider_and_model(chat_request.model)
@@ -333,9 +338,11 @@ async def chat_completions(
# Initialize provider with caching and error handling
try:
provider = get_provider_instance(provider_class)
- ic.configureOutput(prefix='DEBUG| '); ic(f"Using provider instance: {provider_class.__name__}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Using provider instance: {provider_class.__name__}")
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Failed to initialize provider {provider_class.__name__}: {e}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Failed to initialize provider {provider_class.__name__}: {e}")
raise APIError(
f"Failed to initialize provider {provider_class.__name__}: {e}",
HTTP_500_INTERNAL_SERVER_ERROR,
@@ -386,7 +393,8 @@ async def chat_completions(
# Re-raise API errors as-is
raise
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Unexpected error in chat completion {request_id}: {e}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Unexpected error in chat completion {request_id}: {e}")
raise APIError(
f"Internal server error: {str(e)}",
HTTP_500_INTERNAL_SERVER_ERROR,
@@ -407,7 +415,8 @@ async def image_generations(
request_id = f"img-{uuid.uuid4()}"
try:
- ic.configureOutput(prefix='INFO| '); ic(f"Processing image generation request {request_id} for model: {image_request.model}")
+ ic.configureOutput(prefix='INFO| ')
+ ic(f"Processing image generation request {request_id} for model: {image_request.model}")
# Resolve TTI provider and model
provider_class, model_name = resolve_tti_provider_and_model(image_request.model)
@@ -415,7 +424,8 @@ async def image_generations(
# Initialize TTI provider
try:
provider = get_tti_provider_instance(provider_class)
- ic.configureOutput(prefix='DEBUG| '); ic(f"Using TTI provider instance: {provider_class.__name__}")
+ ic.configureOutput(prefix='DEBUG| ')
+ ic(f"Using TTI provider instance: {provider_class.__name__}")
except APIError as e:
# Add helpful footer for provider errors
return JSONResponse(
@@ -429,7 +439,8 @@ async def image_generations(
}
)
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Failed to initialize TTI provider {provider_class.__name__}: {e}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Failed to initialize TTI provider {provider_class.__name__}: {e}")
raise APIError(
f"Failed to initialize TTI provider {provider_class.__name__}: {e}",
HTTP_500_INTERNAL_SERVER_ERROR,
@@ -470,13 +481,15 @@ async def image_generations(
)
elapsed = time.time() - start_time
- ic.configureOutput(prefix='INFO| '); ic(f"Completed image generation request {request_id} in {elapsed:.2f}s")
+ ic.configureOutput(prefix='INFO| ')
+ ic(f"Completed image generation request {request_id} in {elapsed:.2f}s")
return response_data
except APIError:
raise
except Exception as e:
- ic.configureOutput(prefix='ERROR| '); ic(f"Unexpected error in image generation {request_id}: {e}")
+ ic.configureOutput(prefix='ERROR| ')
+ ic(f"Unexpected error in image generation {request_id}: {e}")
raise APIError(
f"Internal server error: {str(e)}",
HTTP_500_INTERNAL_SERVER_ERROR,
@@ -576,47 +589,47 @@ async def websearch(
async def get_search_providers():
"""Get information about all available search providers."""
providers = {}
-
+
# Collect all unique engine names
all_engines = set()
for category_engines in ENGINES.values():
all_engines.update(category_engines.keys())
-
+
for engine_name in sorted(all_engines):
# Find all categories this engine supports
categories = []
for category, engines in ENGINES.items():
if engine_name in engines:
categories.append(category)
-
+
# Get supported parameters based on categories
supported_params = ["q"] # query is always supported
-
+
if "text" in categories or "images" in categories or "news" in categories or "videos" in categories:
supported_params.extend(["max_results", "region", "safesearch"])
-
+
if "suggestions" in categories:
supported_params.extend(["region"])
-
+
if "maps" in categories:
supported_params.extend(["place", "street", "city", "county", "state", "country", "postalcode", "latitude", "longitude", "radius", "max_results"])
-
+
if "translate" in categories:
supported_params.extend(["from_", "to"])
-
+
if "weather" in categories:
supported_params.extend(["language"])
-
+
# Remove duplicates
supported_params = list(set(supported_params))
-
+
providers[engine_name] = {
"name": engine_name,
"categories": sorted(categories),
"supported_types": sorted(categories), # types are the same as categories
"parameters": sorted(supported_params)
}
-
+
return {
"providers": providers,
"total_providers": len(providers)
diff --git a/webscout/server/server.py b/webscout/server/server.py
index faf72c70..161a25f0 100644
--- a/webscout/server/server.py
+++ b/webscout/server/server.py
@@ -13,13 +13,12 @@
import uvicorn
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import RedirectResponse
from fastapi.openapi.docs import get_swagger_ui_html
from starlette.responses import HTMLResponse
-from .config import ServerConfig, AppConfig
-from .routes import Api
+from .config import AppConfig, ServerConfig
from .providers import initialize_provider_map, initialize_tti_provider_map
+from .routes import Api
from .ui_templates import LANDING_PAGE_HTML, SWAGGER_CSS
# Configuration constants
@@ -66,14 +65,14 @@ async def custom_swagger_ui_html():
openapi_url=app.openapi_url,
title=app.title + " - API Documentation",
).body.decode("utf-8")
-
+
# Custom footer and styles
footer_html = """
"""
-
+
# Inject custom CSS and footer
html = html.replace("", f"")
html = html.replace("