diff --git a/changelog.md b/changelog.md index fb003cf6..822cf10a 100644 --- a/changelog.md +++ b/changelog.md @@ -2,16 +2,35 @@ All notable changes to this project will be documented in this file. +## [2025.12.21] - 2025-12-21 + +### 🐛 Fixed +- **fix**: Conducted an extensive codebase cleanup using Ruff, resolving over 200 linting issues and potential bugs. +- **fix**: Standardized error handling by replacing bare `except:` blocks with `except Exception:` or specific exception types across multiple modules (Bing search, GGUF converter, SwiftCLI, and various AI providers). +- **fix**: Resolved numerous `F821 Undefined name` and `F405` errors: + - Restored missing `get_item` method in `YTdownloader.py`. + - Defined missing variables (`result`, `content`, `tool_calls`) in `TextPollinationsAI.py` response processing. + - Fixed missing `ic` imports from `litprinter` in multiple TTS providers (`MurfAI`, `OpenAI.fm`, `Parler`, `Qwen`, `Sherpa`, `FreeTTS`). + - Fixed missing `exceptions` import in `FreeTTS`. + - Resolved undefined `CLI` reference in SwiftCLI `Context` using `TYPE_CHECKING` and explicit imports. + - Added missing type hint imports (`Optional`, `Any`, `Union`, `Generator`, `Response`) across 30+ AI provider modules including Cohere, Gemini, Groq, HuggingFace, and more. + - Fixed undefined `LitAgent` reference in `GitToolkit/utils.py` and missing `Union` in `YTdownloader.py`. + - Resolved `Response` naming conflict in `Ayle.py` by aliasing `curl_cffi` response. +- **fix**: Corrected syntax errors and corrupted logic blocks in `YTdownloader.py` and `iask_search.py`. +- **fix**: Improved project adherence to PEP 8: + - Moved module-level imports to the top of files in `server` and `aihumanizer`. + - Replaced incorrect type comparisons (e.g., `== bool`) with idiomatic `is bool`. + - Split multiple statements on single lines (E701, E702) across the entire project for better readability. +- **refactor**: Replaced star imports (`from ... import *`) with explicit imports in `GitToolkit` and `samurai` provider to eliminate name shadowing and improve static analysis. +- **refactor**: Added dynamic model fetching to both DeepInfra providers (`webscout/Provider/Deepinfra.py`, `webscout/Provider/OPENAI/deepinfra.py`) following Groq provider pattern. Implemented `get_models()` and `update_available_models()` class methods that fetch from `https://api.deepinfra.com/v1/models` API endpoint with fallback to default models on failure. Providers now automatically update their available models list during initialization. +### 🚮 Removed +- **removed**: `yep.py` - Removed the YEPCHAT provider and related files. ## [2025.12.20] - 2025-12-20 ### 📝 Documentation Updates - **docs**: litprinter.md - Completely rewrote documentation to be comprehensive and consistent with other Webscout docs. Added detailed sections for IceCream debugging, Rich Console, Panels & Layouts, Traceback Enhancement, Themes & Styling, Advanced Usage, Integration with Webscout, API Reference, Dependencies, and Supported Python Versions. Enhanced with professional formatting, extensive code examples, and parameter tables. ---- - -## [2025.12.20] - 2025-12-20 - ### ✨ Added - **feat**: webscout/Provider/Nvidia.py - New standalone Nvidia NIM provider with dynamic model fetching and advanced stream sanitization. diff --git a/pyproject.toml b/pyproject.toml index 46a04103..130db3e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,5 +118,7 @@ version = {attr = "webscout.version.__version__"} [tool.ruff] line-length = 100 target-version = "py39" + +[tool.ruff.lint] select = ["E", "F", "W", "I"] -ignore = ["E501"] +ignore = ["E501", "F403", "F401"] diff --git a/webscout/AIauto.py b/webscout/AIauto.py index c5674236..ebeee027 100644 --- a/webscout/AIauto.py +++ b/webscout/AIauto.py @@ -1,36 +1,38 @@ """ This module provides the AUTO provider, which automatically selects and uses an available LLM provider from the webscout library that doesn't require -API keys or cookies. +API keys or cookies. """ -from webscout.AIbase import Provider -from webscout.exceptions import AllProvidersFailure -from typing import Union, Any, Dict, Generator, Optional, List, Tuple, Set, Type +import difflib import importlib +import inspect import pkgutil import random -import inspect -import difflib +from typing import Any, Dict, Generator, List, Optional, Tuple, Type, Union + +from webscout.AIbase import Provider, Response +from webscout.exceptions import AllProvidersFailure + -def load_providers(): +def load_providers() -> Tuple[Dict[str, Type[Provider]], set]: """ Dynamically loads all Provider classes from the `webscout.Provider` package. - + This function iterates through the modules in the `webscout.Provider` package, imports each module, and inspects its attributes to identify classes that inherit from the `Provider` base class. It also identifies providers that require special authentication parameters. - + Returns: - tuple: A tuple containing two elements: - - provider_map (dict): A dictionary mapping uppercase provider names to their classes. + Tuple[Dict[str, Type[Provider]], set]: A tuple containing two elements: + - provider_map (Dict[str, Type[Provider]]): A dictionary mapping uppercase provider names to their classes. - api_key_providers (set): A set of uppercase provider names requiring special authentication. """ - provider_map = {} - api_key_providers = set() + provider_map: Dict[str, Type[Provider]] = {} + api_key_providers: set = set() provider_package = importlib.import_module("webscout.Provider") - + for _, module_name, _ in pkgutil.iter_modules(provider_package.__path__): try: module = importlib.import_module(f"webscout.Provider.{module_name}") @@ -39,7 +41,7 @@ def load_providers(): if isinstance(attr, type) and issubclass(attr, Provider) and attr != Provider: p_name = attr_name.upper() provider_map[p_name] = attr - + if hasattr(attr, "required_auth") and attr.required_auth: api_key_providers.add(p_name) else: @@ -49,7 +51,7 @@ def load_providers(): api_key_providers.add(p_name) except (ValueError, TypeError): pass - except Exception as e: + except Exception: pass return provider_map, api_key_providers @@ -73,15 +75,18 @@ def _get_models_safely(provider_cls: type) -> List[str]: val = getattr(provider_cls, "AVAILABLE_MODELS") if isinstance(val, list): models.extend(val) - + if hasattr(provider_cls, "get_models"): try: - res = provider_cls.get_models() - if isinstance(res, list): - models.extend(res) - except: + # Use getattr to call the class method safely + get_models_method = getattr(provider_cls, "get_models") + if callable(get_models_method): + res = get_models_method() + if isinstance(res, list): + models.extend(res) + except Exception: pass - except: + except Exception: pass return list(set(models)) @@ -91,7 +96,7 @@ class AUTO(Provider): """ An automatic provider that intelligently selects and utilizes an available LLM provider from the webscout library. - + It cycles through available free providers until one successfully processes the request. Excludes providers requiring API keys or cookies by default. @@ -99,23 +104,23 @@ class AUTO(Provider): def __init__( self, model: str = "auto", - api_key: str = None, + api_key: Optional[str] = None, is_conversation: bool = True, max_tokens: int = 600, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, - exclude: Optional[list[str]] = None, + act: Optional[str] = None, + exclude: Optional[List[str]] = None, print_provider_info: bool = False, **kwargs: Any, ): """ Initializes the AUTO provider, setting up the parameters for provider selection and request handling. - + This constructor initializes the AUTO provider with various configuration options, including conversation settings, request limits, and provider exclusions. @@ -157,7 +162,7 @@ def __init__( def last_response(self) -> dict[str, Any]: """ Retrieves the last response dictionary from the successfully used provider. - + Returns: dict[str, Any]: The last response dictionary, or an empty dictionary if no provider has been used yet. """ @@ -167,7 +172,7 @@ def last_response(self) -> dict[str, Any]: def conversation(self) -> object: """ Retrieves the conversation object from the successfully used provider. - + Returns: object: The conversation object, or None if no provider has been used yet. """ @@ -188,14 +193,14 @@ def _fuzzy_resolve_provider_and_model( model (str): The model name to search for. Returns: - Optional[Tuple[Type[Provider], str]]: A tuple containing the provider class + Optional[Tuple[Type[Provider], str]]: A tuple containing the provider class and the resolved model name, or None if no match is found. """ available = [ (name, cls) for name, cls in provider_map.items() if name not in self.exclude ] - + if not self.api_key: available = [p for p in available if p[0] not in api_key_providers] @@ -261,7 +266,7 @@ def _resolve_provider_and_model( (name, cls) for name, cls in provider_map.items() if name not in self.exclude ] - + if not self.api_key: available = [p for p in available if p[0] not in api_key_providers] @@ -281,13 +286,13 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, **kwargs: Any, - ) -> Union[Dict, Generator]: + ) -> Response: """ Sends the prompt to available providers, attempting to get a response from each until one succeeds. - + This method iterates through a prioritized list of available providers based on the requested model and attempts to send the prompt to each provider until a successful response is received. @@ -316,20 +321,20 @@ def ask( queue = [] if resolved_provider: queue.append((resolved_provider.__name__.upper(), resolved_provider, resolved_model)) - + all_available = [ (name, cls) for name, cls in provider_map.items() if name not in self.exclude and (resolved_provider is None or cls != resolved_provider) ] - + if not self.api_key: all_available = [p for p in all_available if p[0] not in api_key_providers] - + random.shuffle(all_available) model_prio = [] others = [] - + for name, cls in all_available: p_models = _get_models_safely(cls) if resolved_model != "auto" and resolved_model in p_models: @@ -342,7 +347,7 @@ def ask( m = random.choice(p_models) queue_model = m if m else "auto" others.append((name, cls, queue_model)) - + queue.extend(model_prio) queue.extend(others) @@ -360,12 +365,12 @@ def ask( "history_offset": self.history_offset, "act": self.act, } - + if 'model' in sig: init_kwargs['model'] = model_to_use if 'api_key' in sig and self.api_key: init_kwargs['api_key'] = self.api_key - + for k, v in self.kwargs.items(): if k in sig or any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.values()): init_kwargs[k] = v @@ -385,7 +390,7 @@ def ask( except Exception: continue - def chained_gen(): + def chained_gen() -> Any: if self.print_provider_info: model = getattr(self.provider, "model", None) provider_class_name = self.provider.__class__.__name__ @@ -423,9 +428,10 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[str, Generator[str, None, None]]: + **kwargs: Any, + ) -> Union[str, Generator[str, None, None]]: """ Provides a simplified chat interface, returning the message string or a generator of message strings. @@ -444,13 +450,13 @@ def chat( else: return self._chat_non_stream(prompt, optimizer, conversationally) - def _chat_stream(self, prompt: str, optimizer: str, conversationally: bool) -> Generator[str, None, None]: + def _chat_stream(self, prompt: str, optimizer: Optional[str], conversationally: bool) -> Generator[str, None, None]: """ Internal helper for streaming chat responses. Args: prompt (str): The user's prompt. - optimizer (str): Name of the optimizer. + optimizer (Optional[str]): Name of the optimizer. conversationally (bool): Whether to apply optimizer conversationally. Yields: @@ -462,16 +468,19 @@ def _chat_stream(self, prompt: str, optimizer: str, conversationally: bool) -> G optimizer=optimizer, conversationally=conversationally, ) - for chunk in response: - yield self.get_message(chunk) + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes, dict)): + for chunk in response: + yield self.get_message(chunk) + elif isinstance(response, dict): + yield self.get_message(response) - def _chat_non_stream(self, prompt: str, optimizer: str, conversationally: bool) -> str: + def _chat_non_stream(self, prompt: str, optimizer: Optional[str], conversationally: bool) -> str: """ Internal helper for non-streaming chat responses. Args: prompt (str): The user's prompt. - optimizer (str): Name of the optimizer. + optimizer (Optional[str]): Name of the optimizer. conversationally (bool): Whether to apply optimizer conversationally. Returns: @@ -483,23 +492,27 @@ def _chat_non_stream(self, prompt: str, optimizer: str, conversationally: bool) optimizer=optimizer, conversationally=conversationally, ) - return self.get_message(response) + if isinstance(response, dict): + return self.get_message(response) + return str(response) - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """ Extracts the message text from the provider's response dictionary. Args: - response (dict): The response dictionary obtained from the `ask` method. + response (Response): The response obtained from the `ask` method. Returns: str: The extracted message string. """ assert self.provider is not None, "Chat with AI first" + if not isinstance(response, dict): + return str(response) return self.provider.get_message(response) - + if __name__ == "__main__": auto = AUTO(print_provider_info=True) response = auto.chat("Hello, how are you?", stream=True) for chunk in response: - print(chunk, end="", flush=True) \ No newline at end of file + print(chunk, end="", flush=True) diff --git a/webscout/AIbase.py b/webscout/AIbase.py index 39002724..5068bc38 100644 --- a/webscout/AIbase.py +++ b/webscout/AIbase.py @@ -1,20 +1,21 @@ from abc import ABC, abstractmethod from pathlib import Path -from typing import Dict, List, Union, Generator, Optional, Any +from typing import Any, Dict, Generator, List, Optional, Union + from typing_extensions import TypeAlias # Type aliases for better readability -Response: TypeAlias = dict[str, Union[str, bool, None]] +Response: TypeAlias = Union[Dict[str, Any], Generator[Any, None, None]] class SearchResponse: """A wrapper class for search API responses. - + This class automatically converts response objects to their text representation when printed or converted to string. - + Attributes: text (str): The text content of the response - + Example: >>> response = SearchResponse("Hello, world!") >>> print(response) @@ -24,10 +25,10 @@ class SearchResponse: """ def __init__(self, text: str): self.text = text - + def __str__(self): return self.text - + def __repr__(self): return self.text @@ -35,6 +36,9 @@ class AIProviderError(Exception): pass class Provider(ABC): + def __init__(self, *args, **kwargs): + self.last_response: Dict[str, Any] = {} + self.conversation: Optional[Any] = None @abstractmethod def ask( @@ -54,7 +58,8 @@ def chat( stream: bool = False, optimizer: Optional[str] = None, conversationally: bool = False, - ) -> str: + **kwargs: Any, + ) -> Union[str, Generator[str, None, None]]: raise NotImplementedError("Method needs to be implemented in subclass") @abstractmethod @@ -64,7 +69,7 @@ def get_message(self, response: Response) -> str: class TTSProvider(ABC): @abstractmethod - def tts(self, text: str, voice: str = None, verbose: bool = False) -> str: + def tts(self, text: str, voice: Optional[str] = None, verbose: bool = False) -> str: """Convert text to speech and save to a temporary file. Args: @@ -77,7 +82,7 @@ def tts(self, text: str, voice: str = None, verbose: bool = False) -> str: """ raise NotImplementedError("Method needs to be implemented in subclass") - def save_audio(self, audio_file: str, destination: str = None, verbose: bool = False) -> str: + def save_audio(self, audio_file: str, destination: Optional[str] = None, verbose: bool = False) -> str: """Save audio to a specific destination. Args: @@ -88,10 +93,10 @@ def save_audio(self, audio_file: str, destination: str = None, verbose: bool = F Returns: str: Path to the saved audio file """ - import shutil import os - from pathlib import Path + import shutil import time + from pathlib import Path source_path = Path(audio_file) @@ -114,7 +119,7 @@ def save_audio(self, audio_file: str, destination: str = None, verbose: bool = F return destination - def stream_audio(self, text: str, voice: str = None, chunk_size: int = 1024, verbose: bool = False) -> Generator[bytes, None, None]: + def stream_audio(self, text: str, voice: Optional[str] = None, chunk_size: int = 1024, verbose: bool = False) -> Generator[bytes, None, None]: """Stream audio in chunks. Args: @@ -166,26 +171,27 @@ def transcribe_from_url(self, audio_url: str, **kwargs) -> Dict[str, Any]: class AISearch(ABC): """Abstract base class for AI-powered search providers. - + This class defines the interface for AI search providers that can perform web searches and return AI-generated responses based on search results. - + All search providers should inherit from this class and implement the required methods. """ - + @abstractmethod def search( self, prompt: str, stream: bool = False, raw: bool = False, - ) -> Union[SearchResponse, Generator[Union[Dict[str, str], SearchResponse], None, None]]: + **kwargs: Any, + ) -> Union[SearchResponse, Generator[Union[Dict[str, str], SearchResponse], None, None], List[Any], Dict[str, Any], str]: """Search using the provider's API and get AI-generated responses. - + This method sends a search query to the provider and returns the AI-generated response. It supports both streaming and non-streaming modes, as well as raw response format. - + Args: prompt (str): The search query or prompt to send to the API. stream (bool, optional): If True, yields response chunks as they arrive. @@ -193,12 +199,12 @@ def search( raw (bool, optional): If True, returns raw response dictionaries. If False, returns SearchResponse objects that convert to text automatically. Defaults to False. - + Returns: - Union[SearchResponse, Generator[Union[Dict[str, str], SearchResponse], None, None]]: + Union[SearchResponse, Generator[Union[Dict[str, str], SearchResponse], None, None]]: - If stream=False: Returns complete response as SearchResponse object - If stream=True: Yields response chunks as either Dict or SearchResponse objects - + Raises: APIConnectionError: If the API request fails """ diff --git a/webscout/AIutel.py b/webscout/AIutel.py index 0f182351..418dafbe 100644 --- a/webscout/AIutel.py +++ b/webscout/AIutel.py @@ -1,18 +1,19 @@ -from .sanitize import * # noqa: E402,F401 -from .conversation import Conversation # noqa: E402,F401 +import functools +import time + +# --- Utility Decorators --- +from typing import Callable +from .conversation import Conversation # noqa: E402,F401 from .optimizers import Optimizers # noqa: E402,F401 from .prompt_manager import AwesomePrompts # noqa: E402,F401 +from .sanitize import * # noqa: E402, F401, F403 -# --- Utility Decorators --- -from typing import Callable -import time -import functools def timeIt(func: Callable): """ Decorator to measure execution time of a function (sync or async). - Prints: - Execution time for '{func.__name__}' : {elapsed:.6f} Seconds. + Prints: - Execution time for '{func.__name__}' : {elapsed:.6f} Seconds. """ import asyncio GREEN_BOLD = "\033[1;92m" @@ -23,7 +24,8 @@ def sync_wrapper(*args, **kwargs): result = func(*args, **kwargs) end_time = time.time() print() - print(f"{GREEN_BOLD}- Execution time for '{func.__name__}' : {end_time - start_time:.6f} Seconds. {RESET}\n") + func_name = getattr(func, "__name__", str(func)) + print(f"{GREEN_BOLD}- Execution time for '{func_name}' : {end_time - start_time:.6f} Seconds. {RESET}\n") return result @functools.wraps(func) @@ -32,7 +34,8 @@ async def async_wrapper(*args, **kwargs): result = await func(*args, **kwargs) end_time = time.time() print() - print(f"{GREEN_BOLD}- Execution time for '{func.__name__}' : {end_time - start_time:.6f} Seconds. {RESET}\n") + func_name = getattr(func, "__name__", str(func)) + print(f"{GREEN_BOLD}- Execution time for '{func_name}' : {end_time - start_time:.6f} Seconds. {RESET}\n") return result if asyncio.iscoroutinefunction(func): @@ -59,6 +62,7 @@ def wrapper(*args, **kwargs): if last_exc is not None: raise last_exc else: - raise RuntimeError(f"Function {func.__name__} failed after {retries} retries with no exception recorded") + func_name = getattr(func, "__name__", str(func)) + raise RuntimeError(f"Function {func_name} failed after {retries} retries with no exception recorded") return wrapper return decorator diff --git a/webscout/Bard.py b/webscout/Bard.py index 27d6c1a4..93f52dbd 100644 --- a/webscout/Bard.py +++ b/webscout/Bard.py @@ -7,24 +7,32 @@ from datetime import datetime from enum import Enum from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union try: - import trio # type: ignore + import trio # type: ignore # noqa: F401 except ImportError: pass from curl_cffi import CurlError from curl_cffi.requests import AsyncSession - from pydantic import BaseModel, field_validator - from requests.exceptions import HTTPError, RequestException, Timeout - from rich.console import Console console = Console() +class AskResponse(TypedDict): + content: str + conversation_id: str + response_id: str + factualityQueries: Optional[List] + textQuery: str + choices: List[Dict[str, Union[str, List]]] + images: List[Dict[str, str]] + error: bool + + class Endpoint(Enum): """ @@ -88,7 +96,7 @@ class Model(Enum): False, ) - def __init__(self, name, header, advanced_only): + def __init__(self, name: str, header: Dict[str, str], advanced_only: bool): """ Initialize a Model enum member. @@ -199,7 +207,7 @@ def load_cookies(cookie_path: str) -> Tuple[str, str]: session_auth2 = next((item['value'] for item in cookies if item['name'].upper() == '__SECURE-1PSIDTS'), None) if not session_auth1 or not session_auth2: - raise StopIteration("Required cookies (__Secure-1PSID or __Secure-1PSIDTS) not found.") + raise ValueError("Required cookies (__Secure-1PSID or __Secure-1PSIDTS) not found.") return session_auth1, session_auth2 except FileNotFoundError: @@ -208,7 +216,7 @@ def load_cookies(cookie_path: str) -> Tuple[str, str]: raise Exception("Invalid JSON format in the cookie file.") except StopIteration as e: raise Exception(f"{e} Check the cookie file format and content.") - except Exception as e: + except Exception as e: raise Exception(f"An unexpected error occurred while loading cookies: {e}") @@ -228,13 +236,13 @@ class Chatbot: def __init__( self, cookie_path: str, - proxy: Optional[Union[str, Dict[str, str]]] = None, + proxy: Optional[Union[str, Dict[str, str]]] = None, timeout: int = 20, model: Model = Model.UNSPECIFIED, impersonate: str = "chrome110" ): try: - self.loop = asyncio.get_running_loop() + self.loop: asyncio.AbstractEventLoop = asyncio.get_running_loop() except RuntimeError: self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) @@ -259,7 +267,7 @@ def load_conversation(self, file_path: str, conversation_name: str) -> bool: self.async_chatbot.load_conversation(file_path, conversation_name) ) - def ask(self, message: str, image: Optional[Union[bytes, str, Path]] = None) -> dict: + def ask(self, message: str, image: Optional[Union[bytes, str, Path]] = None) -> AskResponse: return self.loop.run_until_complete(self.async_chatbot.ask(message, image=image)) class AsyncChatbot: @@ -292,37 +300,37 @@ class AsyncChatbot: "conversation_id", "response_id", "choice_id", - "proxy", - "proxies_dict", + "proxy", + "proxies_dict", "secure_1psidts", "secure_1psid", "session", "timeout", "model", - "impersonate", + "impersonate", ] def __init__( self, secure_1psid: str, secure_1psidts: str, - proxy: Optional[Union[str, Dict[str, str]]] = None, + proxy: Optional[Union[str, Dict[str, str]]] = None, timeout: int = 20, model: Model = Model.UNSPECIFIED, - impersonate: str = "chrome110", + impersonate: str = "chrome110", ): headers = Headers.GEMINI.value.copy() if model != Model.UNSPECIFIED: headers.update(model.model_header) - self._reqid = int("".join(random.choices(string.digits, k=7))) - self.proxy = proxy - self.impersonate = impersonate + self._reqid = int("".join(random.choices(string.digits, k=7))) + self.proxy = proxy + self.impersonate = impersonate self.proxies_dict = None if isinstance(proxy, str): - self.proxies_dict = {"http": proxy, "https": proxy} + self.proxies_dict = {"http": proxy, "https": proxy} elif isinstance(proxy, dict): - self.proxies_dict = proxy + self.proxies_dict = proxy self.conversation_id = "" self.response_id = "" @@ -330,41 +338,54 @@ def __init__( self.secure_1psid = secure_1psid self.secure_1psidts = secure_1psidts - self.session = AsyncSession( + self.session: AsyncSession = AsyncSession( headers=headers, cookies={"__Secure-1PSID": secure_1psid, "__Secure-1PSIDTS": secure_1psidts}, - proxies=self.proxies_dict, + proxies=self.proxies_dict if self.proxies_dict else None, timeout=timeout, - impersonate=self.impersonate + impersonate=self.impersonate if self.impersonate else None ) - self.timeout = timeout + self.timeout = timeout self.model = model - self.SNlM0e = None + self.SNlM0e = None @classmethod async def create( cls, secure_1psid: str, secure_1psidts: str, - proxy: Optional[Union[str, Dict[str, str]]] = None, + proxy: Optional[Union[str, Dict[str, str]]] = None, timeout: int = 20, model: Model = Model.UNSPECIFIED, - impersonate: str = "chrome110", + impersonate: str = "chrome110", ) -> "AsyncChatbot": """ Factory method to create and initialize an AsyncChatbot instance. Fetches the necessary SNlM0e value asynchronously. """ - instance = cls(secure_1psid, secure_1psidts, proxy, timeout, model, impersonate) + instance = cls(secure_1psid, secure_1psidts, proxy, timeout, model, impersonate) try: instance.SNlM0e = await instance.__get_snlm0e() except Exception as e: console.log(f"[red]Error during AsyncChatbot initialization (__get_snlm0e): {e}[/red]", style="bold red") - await instance.session.close() - raise + await instance.session.close() + raise return instance + def _error_response(self, message: str) -> AskResponse: + """Helper to create a consistent error response.""" + return { + "content": message, + "conversation_id": getattr(self, "conversation_id", ""), + "response_id": getattr(self, "response_id", ""), + "factualityQueries": [], + "textQuery": "", + "choices": [], + "images": [], + "error": True + } + async def save_conversation(self, file_path: str, conversation_name: str) -> None: conversations = await self.load_conversations(file_path) conversation_data = { @@ -374,18 +395,18 @@ async def save_conversation(self, file_path: str, conversation_name: str) -> Non "response_id": self.response_id, "choice_id": self.choice_id, "SNlM0e": self.SNlM0e, - "model_name": self.model.model_name, - "timestamp": datetime.now().isoformat(), + "model_name": self.model.model_name, + "timestamp": datetime.now().isoformat(), } found = False for i, conv in enumerate(conversations): if conv.get("conversation_name") == conversation_name: - conversations[i] = conversation_data + conversations[i] = conversation_data found = True break if not found: - conversations.append(conversation_data) + conversations.append(conversation_data) try: Path(file_path).parent.mkdir(parents=True, exist_ok=True) @@ -430,7 +451,7 @@ async def load_conversation(self, file_path: str, conversation_name: str) -> boo console.log(f"[yellow]Conversation '{conversation_name}' not found in {file_path}[/yellow]") return False - async def __get_snlm0e(self): + async def __get_snlm0e(self) -> str: """Fetches the SNlM0e value required for API requests using curl_cffi.""" if not self.secure_1psid: raise ValueError("__Secure-1PSID cookie is required.") @@ -438,9 +459,9 @@ async def __get_snlm0e(self): try: resp = await self.session.get( Endpoint.INIT.value, - timeout=self.timeout + timeout=self.timeout ) - resp.raise_for_status() + resp.raise_for_status() if "Sign in to continue" in resp.text or "accounts.google.com" in str(resp.url): raise PermissionError("Authentication failed. Cookies might be invalid or expired. Please update them.") @@ -462,17 +483,17 @@ async def __get_snlm0e(self): return snlm0e_match.group(1) - except Timeout as e: + except Timeout as e: raise TimeoutError(f"Request timed out while fetching SNlM0e: {e}") from e - except (RequestException, CurlError) as e: + except (RequestException, CurlError) as e: raise ConnectionError(f"Network error while fetching SNlM0e: {e}") from e - except HTTPError as e: + except HTTPError as e: if e.response.status_code == 401 or e.response.status_code == 403: raise PermissionError(f"Authentication failed (status {e.response.status_code}). Check cookies. {e}") from e else: raise Exception(f"HTTP error {e.response.status_code} while fetching SNlM0e: {e}") from e - async def __rotate_cookies(self): + async def __rotate_cookies(self) -> None: """Rotates the __Secure-1PSIDTS cookie.""" try: response = await self.session.post( @@ -492,7 +513,7 @@ async def __rotate_cookies(self): raise - async def ask(self, message: str, image: Optional[Union[bytes, str, Path]] = None) -> dict: + async def ask(self, message: str, image: Optional[Union[bytes, str, Path]] = None) -> AskResponse: """ Sends a message to Google Gemini and returns the response using curl_cffi. @@ -521,7 +542,7 @@ async def ask(self, message: str, image: Optional[Union[bytes, str, Path]] = Non console.log(f"Image uploaded successfully. ID: {image_upload_id}") except Exception as e: console.log(f"[red]Error uploading image: {e}[/red]") - return {"content": f"Error uploading image: {e}", "error": True} + return self._error_response(f"Error uploading image: {e}") if image_upload_id: message_struct = [ @@ -585,7 +606,7 @@ async def ask(self, message: str, image: Optional[Union[bytes, str, Path]] = Non continue if not body: - return {"content": "Failed to parse response body. No valid data found.", "error": True} + return self._error_response("Failed to parse response body. No valid data found.") try: content = "" @@ -705,7 +726,7 @@ async def ask(self, message: str, image: Optional[Union[bytes, str, Path]] = Non all_images = images + generated_images - results = { + results: AskResponse = { "content": content, "conversation_id": conversation_id, "response_id": response_id, @@ -725,23 +746,23 @@ async def ask(self, message: str, image: Optional[Union[bytes, str, Path]] = Non except (IndexError, TypeError) as e: console.log(f"[red]Error extracting data from response: {e}[/red]") - return {"content": f"Error extracting data from response: {e}", "error": True} + return self._error_response(f"Error extracting data from response: {e}") except json.JSONDecodeError as e: console.log(f"[red]Error parsing JSON response: {e}[/red]") - return {"content": f"Error parsing JSON response: {e}. Response: {resp.text[:200]}...", "error": True} + return self._error_response(f"Error parsing JSON response: {e}. Response: {resp.text[:200]}...") except Timeout as e: console.log(f"[red]Request timed out: {e}[/red]") - return {"content": f"Request timed out: {e}", "error": True} + return self._error_response(f"Request timed out: {e}") except (RequestException, CurlError) as e: console.log(f"[red]Network error: {e}[/red]") - return {"content": f"Network error: {e}", "error": True} + return self._error_response(f"Network error: {e}") except HTTPError as e: console.log(f"[red]HTTP error {e.response.status_code}: {e}[/red]") - return {"content": f"HTTP error {e.response.status_code}: {e}", "error": True} + return self._error_response(f"HTTP error {e.response.status_code}: {e}") except Exception as e: console.log(f"[red]An unexpected error occurred during ask: {e}[/red]", style="bold red") - return {"content": f"An unexpected error occurred: {e}", "error": True} + return self._error_response(f"An unexpected error occurred: {e}") @@ -763,10 +784,10 @@ class Image(BaseModel): proxy: Optional[Union[str, Dict[str, str]]] = None impersonate: str = "chrome110" - def __str__(self): + def __str__(self) -> str: return f"{self.title}({self.url}) - {self.alt}" - def __repr__(self): + def __repr__(self) -> Any: short_url = self.url if len(self.url) <= 50 else self.url[:20] + "..." + self.url[-20:] short_alt = self.alt[:30] + "..." if len(self.alt) > 30 else self.alt return f"Image(title='{self.title}', url='{short_url}', alt='{short_alt}')" @@ -775,7 +796,7 @@ async def save( self, path: str = "downloaded_images", filename: Optional[str] = None, - cookies: Optional[dict] = None, + cookies: Optional[Dict[str, str]] = None, verbose: bool = False, skip_invalid_filename: bool = True, ) -> Optional[str]: @@ -840,7 +861,7 @@ async def save( cookies=cookies, proxies=proxies_dict, impersonate=self.impersonate - + ) as client: if verbose: console.log(f"Attempting to download image from: {self.url}") @@ -883,7 +904,35 @@ class WebImage(Image): Returned when asking Gemini to "SEND an image of [something]". """ - pass + async def save( + self, + path: str = "downloaded_images", + filename: Optional[str] = None, + cookies: Optional[Dict[str, str]] = None, + verbose: bool = False, + skip_invalid_filename: bool = True, + ) -> Optional[str]: + """ + Save the image to disk using curl_cffi. + Parameters: + path: str, optional + Directory to save the image (default "downloaded_images"). + filename: str, optional + Filename to use; if not provided, inferred from URL. + cookies: dict, optional + Cookies used for the image request. + verbose: bool, optional + If True, outputs status messages (default False). + skip_invalid_filename: bool, optional + If True, skips saving if the filename is invalid. + Returns: + Absolute path of the saved image if successful; None if skipped. + Raises: + HTTPError if the network request fails. + RequestException/CurlError for other network errors. + IOError if file writing fails. + """ + return await super().save(path, filename, cookies, verbose, skip_invalid_filename) class GeneratedImage(Image): """ diff --git a/webscout/Extra/GitToolkit/__init__.py b/webscout/Extra/GitToolkit/__init__.py index c3ffaa67..a904ec5b 100644 --- a/webscout/Extra/GitToolkit/__init__.py +++ b/webscout/Extra/GitToolkit/__init__.py @@ -1,10 +1,10 @@ -from .gitapi import * +from .gitapi import GitError, NotFoundError, RateLimitError, Repository, RequestError, User __all__ = [ 'Repository', 'User', 'GitError', - 'RateLimitError', + 'RateLimitError', 'NotFoundError', 'RequestError' ] diff --git a/webscout/Extra/GitToolkit/gitapi/__init__.py b/webscout/Extra/GitToolkit/gitapi/__init__.py index fc758110..ca269cf0 100644 --- a/webscout/Extra/GitToolkit/gitapi/__init__.py +++ b/webscout/Extra/GitToolkit/gitapi/__init__.py @@ -1,10 +1,10 @@ -from .repository import Repository -from .user import User -from .search import GitSearch from .gist import Gist from .organization import Organization +from .repository import Repository +from .search import GitSearch from .trending import Trending -from .utils import GitError, RateLimitError, NotFoundError, RequestError +from .user import User +from .utils import GitError, NotFoundError, RateLimitError, RequestError __all__ = [ 'Repository', @@ -14,7 +14,7 @@ 'Organization', 'Trending', 'GitError', - 'RateLimitError', + 'RateLimitError', 'NotFoundError', 'RequestError' ] diff --git a/webscout/Extra/GitToolkit/gitapi/gist.py b/webscout/Extra/GitToolkit/gitapi/gist.py index 9424c6e2..f56431a9 100644 --- a/webscout/Extra/GitToolkit/gitapi/gist.py +++ b/webscout/Extra/GitToolkit/gitapi/gist.py @@ -1,25 +1,26 @@ -from typing import List, Dict, Any, Optional +from typing import Any, Dict, List, Optional + from .utils import request class Gist: """Class for interacting with GitHub Gists without authentication""" - + BASE_URL = "https://api.github.com/gists" - + def get(self, gist_id: str) -> Dict[str, Any]: """ Get a specific gist by ID. - + Args: gist_id: The gist ID - + Returns: Gist data including files, description, owner, etc. """ url = f"{self.BASE_URL}/{gist_id}" return request(url) - + def list_public( self, since: Optional[str] = None, @@ -28,12 +29,12 @@ def list_public( ) -> List[Dict[str, Any]]: """ List public gists sorted by most recently updated. - + Args: since: Only gists updated after this time (ISO 8601 format) page: Page number per_page: Results per page (max 100) - + Returns: List of public gists """ @@ -41,7 +42,7 @@ def list_public( if since: url += f"&since={since}" return request(url) - + def list_for_user( self, username: str, @@ -51,13 +52,13 @@ def list_for_user( ) -> List[Dict[str, Any]]: """ List public gists for a user. - + Args: username: GitHub username since: Only gists updated after this time (ISO 8601 format) page: Page number per_page: Results per page (max 100) - + Returns: List of user's public gists """ @@ -65,7 +66,7 @@ def list_for_user( if since: url += f"&since={since}" return request(url) - + def get_commits( self, gist_id: str, @@ -74,18 +75,18 @@ def get_commits( ) -> List[Dict[str, Any]]: """ Get commit history for a gist. - + Args: gist_id: The gist ID page: Page number per_page: Results per page (max 100) - + Returns: List of commits with version, user, change_status, committed_at """ url = f"{self.BASE_URL}/{gist_id}/commits?page={page}&per_page={per_page}" return request(url) - + def get_forks( self, gist_id: str, @@ -94,32 +95,32 @@ def get_forks( ) -> List[Dict[str, Any]]: """ List forks of a gist. - + Args: gist_id: The gist ID page: Page number per_page: Results per page (max 100) - + Returns: List of gist forks """ url = f"{self.BASE_URL}/{gist_id}/forks?page={page}&per_page={per_page}" return request(url) - + def get_revision(self, gist_id: str, sha: str) -> Dict[str, Any]: """ Get a specific revision of a gist. - + Args: gist_id: The gist ID sha: The revision SHA - + Returns: Gist data at that specific revision """ url = f"{self.BASE_URL}/{gist_id}/{sha}" return request(url) - + def get_comments( self, gist_id: str, @@ -128,12 +129,12 @@ def get_comments( ) -> List[Dict[str, Any]]: """ Get comments on a gist. - + Args: gist_id: The gist ID page: Page number per_page: Results per page (max 100) - + Returns: List of comments """ diff --git a/webscout/Extra/GitToolkit/gitapi/organization.py b/webscout/Extra/GitToolkit/gitapi/organization.py index 8986c7ab..d6521eda 100644 --- a/webscout/Extra/GitToolkit/gitapi/organization.py +++ b/webscout/Extra/GitToolkit/gitapi/organization.py @@ -1,14 +1,15 @@ -from typing import List, Dict, Any +from typing import Any, Dict, List + from .utils import request class Organization: """Class for interacting with GitHub organization data without authentication""" - + def __init__(self, org: str): """ Initialize organization client. - + Args: org: Organization login name """ @@ -16,19 +17,19 @@ def __init__(self, org: str): raise ValueError("Organization name is required") if not isinstance(org, str): raise ValueError("Organization name must be a string") - + self.org = org.strip() self.base_url = f"https://api.github.com/orgs/{self.org}" - + def get_info(self) -> Dict[str, Any]: """ Get organization information. - + Returns: Organization details including name, description, location, etc. """ return request(self.base_url) - + def get_repos( self, repo_type: str = "all", @@ -39,20 +40,20 @@ def get_repos( ) -> List[Dict[str, Any]]: """ List organization repositories. - + Args: repo_type: Type of repos (all, public, private, forks, sources, member) sort: Sort by (created, updated, pushed, full_name) direction: Sort direction (asc, desc) page: Page number per_page: Results per page (max 100) - + Returns: List of organization repositories """ url = f"{self.base_url}/repos?type={repo_type}&sort={sort}&direction={direction}&page={page}&per_page={per_page}" return request(url) - + def get_public_members( self, page: int = 1, @@ -60,17 +61,17 @@ def get_public_members( ) -> List[Dict[str, Any]]: """ List public members of the organization. - + Args: page: Page number per_page: Results per page (max 100) - + Returns: List of public organization members """ url = f"{self.base_url}/public_members?page={page}&per_page={per_page}" return request(url) - + def get_events( self, page: int = 1, @@ -78,11 +79,11 @@ def get_events( ) -> List[Dict[str, Any]]: """ List public organization events. - + Args: page: Page number per_page: Results per page (max 100) - + Returns: List of public events """ diff --git a/webscout/Extra/GitToolkit/gitapi/repository.py b/webscout/Extra/GitToolkit/gitapi/repository.py index ffbdd3b3..7ae6c316 100644 --- a/webscout/Extra/GitToolkit/gitapi/repository.py +++ b/webscout/Extra/GitToolkit/gitapi/repository.py @@ -1,14 +1,16 @@ -from typing import List, Dict, Any, Optional -from .utils import request +from typing import Any, Dict, List, Optional from urllib.parse import quote +from .utils import request + + class Repository: """Class for interacting with GitHub repositories""" - + def __init__(self, owner: str, repo: str): """ Initialize repository client - + Args: owner: Repository owner repo: Repository name @@ -17,7 +19,7 @@ def __init__(self, owner: str, repo: str): raise ValueError("Owner and repository name are required") if not isinstance(owner, str) or not isinstance(repo, str): raise ValueError("Owner and repository name must be strings") - + self.owner = owner.strip() self.repo = repo.strip() self.base_url = f"https://api.github.com/repos/{self.owner}/{self.repo}" @@ -29,7 +31,7 @@ def get_info(self) -> Dict[str, Any]: def get_commits(self, page: int = 1, per_page: int = 30, sha: str = None) -> List[Dict[str, Any]]: """ Get repository commits - + Args: page: Page number per_page: Items per page @@ -48,7 +50,7 @@ def get_commit(self, sha: str) -> Dict[str, Any]: def get_pull_requests(self, state: str = "all", page: int = 1, per_page: int = 30) -> List[Dict[str, Any]]: """ Get repository pull requests - + Args: state: State of PRs to return (open/closed/all) page: Page number @@ -75,7 +77,7 @@ def get_pull_request_files(self, pr_number: int) -> List[Dict[str, Any]]: def get_issues(self, state: str = "all", page: int = 1, per_page: int = 30, labels: str = None) -> List[Dict[str, Any]]: """ Get repository issues - + Args: state: State of issues to return (open/closed/all) page: Page number @@ -202,10 +204,10 @@ def get_community_profile(self) -> Dict[str, Any]: def get_readme(self, ref: Optional[str] = None) -> Dict[str, Any]: """ Get repository README. - + Args: ref: The name of the commit/branch/tag (default: repo default branch) - + Returns: README content with encoding and download_url """ @@ -217,7 +219,7 @@ def get_readme(self, ref: Optional[str] = None) -> Dict[str, Any]: def get_license(self) -> Dict[str, Any]: """ Get repository license. - + Returns: License information including name, key, spdx_id, and content """ @@ -227,7 +229,7 @@ def get_license(self) -> Dict[str, Any]: def get_topics(self) -> Dict[str, Any]: """ Get repository topics. - + Returns: Dict with 'names' key containing list of topic strings """ @@ -237,12 +239,12 @@ def get_topics(self) -> Dict[str, Any]: def get_forks(self, sort: str = "newest", page: int = 1, per_page: int = 30) -> List[Dict[str, Any]]: """ List repository forks. - + Args: sort: Sort by (newest, oldest, stargazers, watchers) page: Page number per_page: Results per page (max 100) - + Returns: List of forked repositories """ @@ -252,11 +254,11 @@ def get_forks(self, sort: str = "newest", page: int = 1, per_page: int = 30) -> def get_stargazers(self, page: int = 1, per_page: int = 30) -> List[Dict[str, Any]]: """ List users who have starred the repository. - + Args: page: Page number per_page: Results per page (max 100) - + Returns: List of users who starred the repo """ @@ -266,11 +268,11 @@ def get_stargazers(self, page: int = 1, per_page: int = 30) -> List[Dict[str, An def get_watchers(self, page: int = 1, per_page: int = 30) -> List[Dict[str, Any]]: """ List users watching the repository (subscribers). - + Args: page: Page number per_page: Results per page (max 100) - + Returns: List of users watching the repo """ @@ -280,11 +282,11 @@ def get_watchers(self, page: int = 1, per_page: int = 30) -> List[Dict[str, Any] def compare(self, base: str, head: str) -> Dict[str, Any]: """ Compare two commits, branches, or tags. - + Args: base: Base commit/branch/tag head: Head commit/branch/tag - + Returns: Comparison data including commits, files, and diff stats """ @@ -294,11 +296,11 @@ def compare(self, base: str, head: str) -> Dict[str, Any]: def get_events(self, page: int = 1, per_page: int = 30) -> List[Dict[str, Any]]: """ List repository events. - + Args: page: Page number per_page: Results per page (max 100) - + Returns: List of repository events """ diff --git a/webscout/Extra/GitToolkit/gitapi/search.py b/webscout/Extra/GitToolkit/gitapi/search.py index 667c50fc..dc8c18b5 100644 --- a/webscout/Extra/GitToolkit/gitapi/search.py +++ b/webscout/Extra/GitToolkit/gitapi/search.py @@ -1,13 +1,14 @@ -from typing import List, Dict, Any, Optional -from .utils import request +from typing import Any, Dict, Optional from urllib.parse import quote +from .utils import request + class GitSearch: """Class for searching GitHub content without authentication""" - + BASE_URL = "https://api.github.com/search" - + def search_repositories( self, query: str, @@ -18,14 +19,14 @@ def search_repositories( ) -> Dict[str, Any]: """ Search for repositories. - + Args: query: Search query (e.g., "tetris language:python stars:>100") sort: Sort by (stars, forks, help-wanted-issues, updated) order: Sort order (asc, desc) page: Page number per_page: Results per page (max 100) - + Returns: Dict with total_count, incomplete_results, and items """ @@ -33,7 +34,7 @@ def search_repositories( if sort: url += f"&sort={sort}" return request(url) - + def search_users( self, query: str, @@ -44,14 +45,14 @@ def search_users( ) -> Dict[str, Any]: """ Search for users. - + Args: query: Search query (e.g., "tom repos:>42 followers:>1000") sort: Sort by (followers, repositories, joined) order: Sort order (asc, desc) page: Page number per_page: Results per page (max 100) - + Returns: Dict with total_count, incomplete_results, and items """ @@ -59,7 +60,7 @@ def search_users( if sort: url += f"&sort={sort}" return request(url) - + def search_topics( self, query: str, @@ -68,18 +69,18 @@ def search_topics( ) -> Dict[str, Any]: """ Search for topics. - + Args: query: Search query page: Page number per_page: Results per page (max 100) - + Returns: Dict with total_count, incomplete_results, and items """ url = f"{self.BASE_URL}/topics?q={quote(query)}&page={page}&per_page={per_page}" return request(url) - + def search_commits( self, query: str, @@ -90,14 +91,14 @@ def search_commits( ) -> Dict[str, Any]: """ Search for commits. - + Args: query: Search query (e.g., "fix bug repo:owner/repo") sort: Sort by (author-date, committer-date) order: Sort order (asc, desc) page: Page number per_page: Results per page (max 100) - + Returns: Dict with total_count, incomplete_results, and items """ @@ -105,7 +106,7 @@ def search_commits( if sort: url += f"&sort={sort}" return request(url) - + def search_issues( self, query: str, @@ -116,14 +117,14 @@ def search_issues( ) -> Dict[str, Any]: """ Search for issues and pull requests. - + Args: query: Search query (e.g., "bug is:issue is:open label:bug") sort: Sort by (comments, reactions, created, updated) order: Sort order (asc, desc) page: Page number per_page: Results per page (max 100) - + Returns: Dict with total_count, incomplete_results, and items """ @@ -131,7 +132,7 @@ def search_issues( if sort: url += f"&sort={sort}" return request(url) - + def search_labels( self, repository_id: int, @@ -143,7 +144,7 @@ def search_labels( ) -> Dict[str, Any]: """ Search for labels in a repository. - + Args: repository_id: Repository ID to search in query: Search query @@ -151,7 +152,7 @@ def search_labels( order: Sort order (asc, desc) page: Page number per_page: Results per page (max 100) - + Returns: Dict with total_count, incomplete_results, and items """ diff --git a/webscout/Extra/GitToolkit/gitapi/trending.py b/webscout/Extra/GitToolkit/gitapi/trending.py index dc438463..6df74309 100644 --- a/webscout/Extra/GitToolkit/gitapi/trending.py +++ b/webscout/Extra/GitToolkit/gitapi/trending.py @@ -1,8 +1,6 @@ -from typing import List, Dict, Any, Optional -from urllib.request import Request, urlopen -from urllib.error import HTTPError import re -import json +from typing import Any, Dict, List +from urllib.request import Request, urlopen try: from webscout.litagent.agent import LitAgent @@ -14,9 +12,9 @@ class Trending: """Class for getting GitHub trending data (scrapes github.com/trending)""" - + BASE_URL = "https://github.com/trending" - + def get_repositories( self, language: str = "", @@ -25,12 +23,12 @@ def get_repositories( ) -> List[Dict[str, Any]]: """ Get trending repositories. - + Args: language: Programming language filter (e.g., "python", "javascript") since: Time range (daily, weekly, monthly) spoken_language: Spoken language filter (e.g., "en" for English) - + Returns: List of trending repositories with name, description, stars, forks, etc. """ @@ -40,10 +38,10 @@ def get_repositories( url += f"?since={since}" if spoken_language: url += f"&spoken_language_code={spoken_language}" - + html = self._fetch_html(url) return self._parse_repos(html) - + def get_developers( self, language: str = "", @@ -51,11 +49,11 @@ def get_developers( ) -> List[Dict[str, Any]]: """ Get trending developers. - + Args: language: Programming language filter since: Time range (daily, weekly, monthly) - + Returns: List of trending developers with username, name, avatar, repo """ @@ -63,10 +61,10 @@ def get_developers( if language: url += f"/{language}" url += f"?since={since}" - + html = self._fetch_html(url) return self._parse_developers(html) - + def _fetch_html(self, url: str) -> str: """Fetch HTML content from URL.""" headers = { @@ -76,23 +74,23 @@ def _fetch_html(self, url: str) -> str: req = Request(url, headers=headers) response = urlopen(req, timeout=30) return response.read().decode('utf-8') - + def _parse_repos(self, html: str) -> List[Dict[str, Any]]: """Parse trending repositories from HTML.""" repos = [] - + # Find all article elements (repo boxes) - try multiple patterns repo_patterns = [ r'
]*>(.*?)
', r']*class="[^"]*Box-row[^"]*"[^>]*>(.*?)', ] - + repo_matches = [] for pattern in repo_patterns: repo_matches = re.findall(pattern, html, re.DOTALL) if repo_matches: break - + # If no matches with article, try row-based parsing if not repo_matches: # Try to find repo links directly @@ -112,17 +110,17 @@ def _parse_repos(self, html: str) -> List[Dict[str, Any]]: 'forks': 0 }) return repos - + for repo_html in repo_matches: repo = {} - + # Extract repo name (owner/repo) - try multiple patterns name_patterns = [ r'href="/([^"]+)"[^>]*>\s*]*>([^<]+)\s*/\s*]*>([^<]+)', r'href="/([^/]+/[^"]+)"[^>]*class="[^"]*Link[^"]*"', r']*>\s*]*href="/([^"]+)"' ] - + for pattern in name_patterns: name_match = re.search(pattern, repo_html) if name_match: @@ -133,7 +131,7 @@ def _parse_repos(self, html: str) -> List[Dict[str, Any]]: repo['owner'] = parts[0].strip() repo['name'] = parts[1].strip() if len(parts) > 1 else '' break - + # Extract description - try multiple patterns desc_patterns = [ r'

]*>([^<]+)

', @@ -147,7 +145,7 @@ def _parse_repos(self, html: str) -> List[Dict[str, Any]]: break else: repo['description'] = "" - + # Extract language lang_patterns = [ r'([^<]+)', @@ -160,7 +158,7 @@ def _parse_repos(self, html: str) -> List[Dict[str, Any]]: break else: repo['language'] = None - + # Extract stars - multiple patterns stars_patterns = [ r'href="/[^/]+/[^/]+/stargazers"[^>]*>\s*(?:]*>.*?)?\s*([\d,]+)', @@ -174,7 +172,7 @@ def _parse_repos(self, html: str) -> List[Dict[str, Any]]: break else: repo['stars'] = 0 - + # Extract forks forks_patterns = [ r'href="/[^/]+/[^/]+/forks"[^>]*>\s*(?:]*>.*?)?\s*([\d,]+)', @@ -187,44 +185,44 @@ def _parse_repos(self, html: str) -> List[Dict[str, Any]]: break else: repo['forks'] = 0 - + # Extract stars today/this week/this month today_match = re.search(r'([\d,]+)\s+stars?\s+(today|this week|this month)', repo_html) if today_match: repo['stars_period'] = int(today_match.group(1).replace(',', '')) repo['period'] = today_match.group(2) - + if repo.get('full_name'): repos.append(repo) - + return repos - + def _parse_developers(self, html: str) -> List[Dict[str, Any]]: """Parse trending developers from HTML.""" developers = [] - + # Find all article elements (developer boxes) dev_pattern = r'
]*>(.*?)
' dev_matches = re.findall(dev_pattern, html, re.DOTALL) - + for dev_html in dev_matches: dev = {} - + # Extract username username_match = re.search(r'href="/([^"?]+)"[^>]*class="[^"]*Link[^"]*"', dev_html) if username_match: dev['username'] = username_match.group(1).strip() - + # Extract display name name_match = re.search(r'

]*>\s*]*>([^<]+)', dev_html) if name_match: dev['name'] = name_match.group(1).strip() - + # Extract avatar avatar_match = re.search(r']*class="[^"]*avatar[^"]*"[^>]*src="([^"]+)"', dev_html) if avatar_match: dev['avatar'] = avatar_match.group(1) - + # Extract popular repo repo_match = re.search(r']*>\s*]*>([^<]+)', dev_html) if repo_match: @@ -232,8 +230,8 @@ def _parse_developers(self, html: str) -> List[Dict[str, Any]]: 'full_name': repo_match.group(1), 'name': repo_match.group(2).strip() } - + if dev.get('username'): developers.append(dev) - + return developers diff --git a/webscout/Extra/GitToolkit/gitapi/user.py b/webscout/Extra/GitToolkit/gitapi/user.py index ef6cb5f6..ad96bfe4 100644 --- a/webscout/Extra/GitToolkit/gitapi/user.py +++ b/webscout/Extra/GitToolkit/gitapi/user.py @@ -1,13 +1,15 @@ -from typing import List, Dict, Any +from typing import Any, Dict, List + from .utils import request + class User: """Class for interacting with GitHub user data""" - + def __init__(self, username: str): """ Initialize user client - + Args: username: GitHub username """ @@ -15,7 +17,7 @@ def __init__(self, username: str): raise ValueError("Username is required") if not isinstance(username, str): raise ValueError("Username must be a string") - + self.username = username.strip() self.base_url = f"https://api.github.com/users/{self.username}" @@ -26,7 +28,7 @@ def get_profile(self) -> Dict[str, Any]: def get_repositories(self, page: int = 1, per_page: int = 30, repo_type: str = "all") -> List[Dict[str, Any]]: """ Get user's public repositories - + Args: page: Page number per_page: Items per page @@ -103,7 +105,7 @@ def get_gpg_keys(self) -> List[Dict[str, Any]]: def get_social_accounts(self) -> List[Dict[str, Any]]: """ Get user's social accounts. - + Returns: List of social accounts with provider and url """ @@ -113,12 +115,12 @@ def get_social_accounts(self) -> List[Dict[str, Any]]: def get_packages(self, package_type: str = "container", page: int = 1, per_page: int = 30) -> List[Dict[str, Any]]: """ Get user's public packages. - + Args: package_type: Type of package (container, npm, maven, rubygems, nuget, docker) page: Page number per_page: Results per page (max 100) - + Returns: List of user's packages """ diff --git a/webscout/Extra/GitToolkit/gitapi/utils.py b/webscout/Extra/GitToolkit/gitapi/utils.py index 0fd25c38..e9005595 100644 --- a/webscout/Extra/GitToolkit/gitapi/utils.py +++ b/webscout/Extra/GitToolkit/gitapi/utils.py @@ -1,8 +1,9 @@ -from urllib.request import Request, urlopen -from urllib.error import HTTPError import json import time -from typing import Dict, Any +from typing import Any, Dict +from urllib.error import HTTPError +from urllib.request import Request, urlopen + try: from webscout.litagent.agent import LitAgent except ImportError: @@ -29,14 +30,14 @@ class RequestError(GitError): def request(url: str, retry_attempts: int = 3) -> Dict[str, Any]: """ Send a request to GitHub API with retry mechanism - + Args: url: GitHub API endpoint URL retry_attempts: Number of retry attempts - + Returns: Parsed JSON response - + Raises: NotFoundError: If resource not found RateLimitError: If rate limited @@ -46,7 +47,7 @@ def request(url: str, retry_attempts: int = 3) -> Dict[str, Any]: "User-Agent": _USER_AGENT_GENERATOR.random() if _USER_AGENT_GENERATOR else "webscout-gitapi/1.0", "Accept": "application/vnd.github+json" } - + for attempt in range(retry_attempts): try: req = Request(url, headers=headers) @@ -56,7 +57,7 @@ def request(url: str, retry_attempts: int = 3) -> Dict[str, Any]: return json.loads(data) except json.JSONDecodeError as json_err: raise RequestError(f"Invalid JSON response from {url}: {str(json_err)}") - + except HTTPError as e: if e.code == 404: raise NotFoundError(f"Resource not found: {url}") @@ -67,12 +68,12 @@ def request(url: str, retry_attempts: int = 3) -> Dict[str, Any]: continue raise RateLimitError(f"Rate limited after {retry_attempts} attempts") if e.code == 403: - raise RequestError(f"Forbidden: Check your authentication token") + raise RequestError("Forbidden: Check your authentication token") if attempt == retry_attempts - 1: raise RequestError(f"HTTP Error {e.code}: {e.reason}") # Wait before retrying on other HTTP errors time.sleep(1) - + except Exception as e: if attempt == retry_attempts - 1: - raise RequestError(f"Request failed: {str(e)}") \ No newline at end of file + raise RequestError(f"Request failed: {str(e)}") diff --git a/webscout/Extra/YTToolkit/YTdownloader.py b/webscout/Extra/YTToolkit/YTdownloader.py index 5b94663f..53bb822b 100644 --- a/webscout/Extra/YTToolkit/YTdownloader.py +++ b/webscout/Extra/YTToolkit/YTdownloader.py @@ -1,18 +1,21 @@ -from datetime import datetime import json -from webscout.litagent import LitAgent -from time import sleep -import requests -from tqdm import tqdm -from colorama import Fore -from os import makedirs, path, getcwd -from threading import Thread import os import subprocess import sys import tempfile -from webscout.version import __prog__, __version__ -from webscout.swiftcli import CLI, option, argument +from datetime import datetime +from os import getcwd, makedirs, path +from threading import Thread +from time import sleep +from typing import Union + +import requests +from colorama import Fore +from tqdm import tqdm + +from webscout.litagent import LitAgent +from webscout.swiftcli import CLI, argument, option +from webscout.version import __prog__ # Define cache directory using tempfile user_cache_dir = os.path.join(tempfile.gettempdir(), 'webscout') @@ -32,7 +35,8 @@ session.headers.update(headers) -get_excep = lambda e: e.args[1] if len(e.args) > 1 else e +def get_excep(e): + return e.args[1] if len(e.args) > 1 else e appdir = user_cache_dir @@ -58,12 +62,12 @@ def main(*args, **kwargs): try: try: return func(*args, **kwargs) - except KeyboardInterrupt as e: + except KeyboardInterrupt: print() exit(1) except Exception as e: if log: - raise(f"Error - {get_excep(e)}") + raise Exception(f"Error - {get_excep(e)}") if exit_on_error: exit(1) @@ -87,7 +91,7 @@ def post(*args, **kwargs): @staticmethod def add_history(data: dict) -> None: - f"""Adds entry to history + """Adds entry to history :param data: Response of `third query` :type data: dict :rtype: None @@ -103,11 +107,11 @@ def add_history(data: dict) -> None: saved_data.append(data) with open(history_path, "w") as fh: json.dump({__prog__: saved_data}, fh, indent=4) - except Exception as e: + except Exception: pass @staticmethod - def get_history(dump: bool = False) -> list: + def get_history(dump: bool = False) -> Union[list, str]: r"""Loads download history :param dump: (Optional) Return whole history as str :type dump: bool @@ -126,7 +130,7 @@ def get_history(dump: bool = False) -> list: for entry in entries: resp.append(entry.get("vid")) return resp - except Exception as e: + except Exception: return [] @@ -152,7 +156,7 @@ def __get_payload(self): def __str__(self): return """ -{ +{ "page": "search", "status": "ok", "keyword": "happy birthday", @@ -240,7 +244,7 @@ def __str__(self): "q": ".m4a", "q_text": ".m4a (128kbps)", "k": "joVBVdm2xZWhaZWhu6vZ8cXxAl7j4qpyhNhuxgxyU/NQ9919mbX2dYcdevRBnt0=" - }, + }, }, "related": [ { @@ -253,13 +257,9 @@ def __str__(self): ] } ] -} - """ - - def __call__(self, *args, **kwargs): - return self.main(*args, **kwargs) - - def get_item(self, item_no=0): +} + """ + def get_item(self, item_no: int = None): r"""Return specific items on `self.query_one.vitems`""" if self.video_dict: return self.video_dict @@ -490,7 +490,7 @@ def __call__(self, *args, **kwargs): return self.run(*args, **kwargs) def __filter_videos(self, entries: list) -> list: - f"""Filter videos based on keyword + """Filter videos based on keyword :param entries: List containing dict of video id and their titles :type entries: list :rtype: list @@ -510,7 +510,7 @@ def __make_first_query(self): r"""Sets query_one attribute to `self`""" query_one = first_query(self.query) self.__setattr__("query_one", query_one.main(self.timeout)) - if self.query_one.is_link == False: + if not self.query_one.is_link: self.vitems.extend(self.__filter_videos(self.query_one.vitems)) @utils.error_handler(exit_on_error=True) @@ -546,7 +546,7 @@ def __make_second_query(self): if query_2.processed: if query_2.vid in self.dropped: continue - if self.author and not self.author.lower() in query_2.a.lower(): + if self.author and self.author.lower() not in query_2.a.lower(): continue else: yes_download, reason = self.__verify_item(query_2) @@ -579,7 +579,7 @@ def __make_second_query(self): if query_2.processed: if ( self.author - and not self.author.lower() in query_2.a.lower() + and self.author.lower() not in query_2.a.lower() ): continue else: @@ -824,9 +824,8 @@ def save( if any([save_to.startswith("/"), ":" in save_to]) else path.join(getcwd(), dir, filename) ) - try_play_media = ( - lambda: launch_media(third_dict["saved_to"]) if play else None - ) + def try_play_media(): + return (launch_media(third_dict["saved_to"]) if play else None) saving_mode = "ab" if resume else "wb" if progress_bar: if not quiet: @@ -896,14 +895,14 @@ def confirm_from_user(message, default=False): """ valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} - + if default is None: prompt = " [y/n] " elif default: prompt = " [Y/n] " else: prompt = " [y/N] " - + while True: choice = input(message + prompt).lower() if default is not None and choice == '': @@ -933,19 +932,19 @@ def download(query, author, timeout, confirm, unique, thread, format, quality, l # Create handler with parsed arguments handler = Handler( - query=query, - author=author, - timeout=timeout, - confirm=confirm, - unique=unique, + query=query, + author=author, + timeout=timeout, + confirm=confirm, + unique=unique, thread=thread ) # Run download process handler.auto_save( - format=format, - quality=quality, - limit=limit, + format=format, + quality=quality, + limit=limit, keyword=keyword ) @@ -954,4 +953,4 @@ def main(): app.run() if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/webscout/Extra/YTToolkit/__init__.py b/webscout/Extra/YTToolkit/__init__.py index 407afc6e..5a96883a 100644 --- a/webscout/Extra/YTToolkit/__init__.py +++ b/webscout/Extra/YTToolkit/__init__.py @@ -1,3 +1,3 @@ -from .YTdownloader import * from .transcriber import * -from .ytapi import * \ No newline at end of file +from .ytapi import * +from .YTdownloader import * diff --git a/webscout/Extra/YTToolkit/transcriber.py b/webscout/Extra/YTToolkit/transcriber.py index 8495a1fd..06f94bc3 100644 --- a/webscout/Extra/YTToolkit/transcriber.py +++ b/webscout/Extra/YTToolkit/transcriber.py @@ -6,16 +6,29 @@ """ -import requests -import http.cookiejar as cookiejar -import json -from xml.etree import ElementTree -import re -import html -from typing import List, Dict, Union, Optional +import html +import http.cookiejar as cookiejar +import re +from concurrent.futures import ThreadPoolExecutor from functools import lru_cache -from concurrent.futures import ThreadPoolExecutor -from webscout.exceptions import * +from typing import Dict, List, Optional, Union +from xml.etree import ElementTree + +import requests + +from webscout.exceptions import ( + CookiePathInvalidError, + FailedToCreateConsentCookieError, + InvalidVideoIdError, + NoTranscriptFoundError, + NotTranslatableError, + TooManyRequestsError, + TranscriptRetrievalError, + TranscriptsDisabledError, + TranslationLanguageNotAvailableError, + VideoUnavailableError, + YouTubeRequestFailedError, +) from webscout.litagent import LitAgent # YouTube API settings @@ -27,15 +40,15 @@ class YTTranscriber: """Transcribe YouTube videos with style! 🎤 - + >>> transcript = YTTranscriber.get_transcript('https://youtu.be/dQw4w9WgXcQ') >>> print(transcript[0]['text']) 'Never gonna give you up' """ - + _session = None _executor = ThreadPoolExecutor(max_workers=MAX_WORKERS) - + @classmethod def _get_session(cls): if cls._session is None: @@ -74,17 +87,17 @@ def get_transcript(cls, video_url: str, languages: Optional[str] = 'en', """ video_id = cls._extract_video_id(video_url) http_client = cls._get_session() - + if proxies: http_client.proxies.update(proxies) - + if cookies: cls._load_cookies(cookies, video_id) transcript_list = TranscriptListFetcher(http_client).fetch(video_id) language_codes = [languages] if languages else None transcript = transcript_list.find_transcript(language_codes) - + return transcript.fetch(preserve_formatting) @staticmethod @@ -96,15 +109,15 @@ def _extract_video_id(video_url: str) -> str: r'youtube\.com\/embed\/([0-9A-Za-z_-]{11})', r'youtube\.com\/shorts\/([0-9A-Za-z_-]{11})' ] - + for pattern in patterns: match = re.search(pattern, video_url) if match: return match.group(1) - + if re.match(r'^[0-9A-Za-z_-]{11}$', video_url): return video_url - + raise InvalidVideoIdError(video_url) @staticmethod @@ -139,7 +152,7 @@ def _fetch_captions_json(self, video_id: str) -> dict: # First get the HTML to extract the API key video_html = self._fetch_video_html(video_id) api_key = self._extract_innertube_api_key(video_html, video_id) - + # Use InnerTube API to get video data innertube_data = self._fetch_innertube_data(video_id, api_key) return self._extract_captions_from_innertube(innertube_data, video_id) @@ -150,11 +163,11 @@ def _extract_innertube_api_key(self, html_content: str, video_id: str) -> str: match = re.search(pattern, html_content) if match and len(match.groups()) == 1: return match.group(1) - + # Check for IP block if 'class="g-recaptcha"' in html_content: raise TooManyRequestsError(video_id) - + raise TranscriptRetrievalError(video_id, "Could not extract InnerTube API key") def _fetch_innertube_data(self, video_id: str, api_key: str) -> dict: @@ -173,13 +186,13 @@ def _extract_captions_from_innertube(self, innertube_data: dict, video_id: str) # Check playability status playability_status = innertube_data.get("playabilityStatus", {}) status = playability_status.get("status") - + if status == "ERROR": reason = playability_status.get("reason", "Unknown error") if "unavailable" in reason.lower(): raise VideoUnavailableError(video_id) raise TranscriptRetrievalError(video_id, reason) - + if status == "LOGIN_REQUIRED": reason = playability_status.get("reason", "") if "bot" in reason.lower(): @@ -187,14 +200,14 @@ def _extract_captions_from_innertube(self, innertube_data: dict, video_id: str) if "age" in reason.lower() or "inappropriate" in reason.lower(): raise TranscriptRetrievalError(video_id, "Video is age-restricted") raise TranscriptRetrievalError(video_id, reason or "Login required") - + # Get captions captions = innertube_data.get("captions", {}) captions_json = captions.get("playerCaptionsTracklistRenderer") - + if captions_json is None or "captionTracks" not in captions_json: raise TranscriptsDisabledError(video_id) - + return captions_json def _create_consent_cookie(self, html_content, video_id): @@ -218,7 +231,7 @@ def _fetch_html(self, video_id): class TranscriptList: - """ + """ >>> transcript_list = TranscriptList.build(http_client, video_id, captions_json) >>> transcript = transcript_list.find_transcript(['en']) >>> print(transcript) @@ -314,7 +327,7 @@ def find_transcript(self, language_codes): Finds a transcript for a given language code. If no language is provided, it will return the first available transcript. - :param language_codes: A list of language codes in a descending priority. + :param language_codes: A list of language codes in a descending priority. :type languages: list[str] :return: the found Transcript :rtype Transcript: @@ -337,7 +350,7 @@ def find_generated_transcript(self, language_codes): it fails to do so. :type languages: list[str] :return: the found Transcript - :rtype Transcript: + :rtype Transcript: :raises: NoTranscriptFound """ if not language_codes: @@ -358,7 +371,7 @@ def find_manually_created_transcript(self, language_codes): it fails to do so. :type languages: list[str] :return: the found Transcript - :rtype Transcript: + :rtype Transcript: :raises: NoTranscriptFound """ if not language_codes: @@ -409,7 +422,7 @@ def _get_language_description(self, transcript_strings): class Transcript: """Your personal transcript handler! 🎭 - + >>> transcript = transcript_list.find_transcript(['en']) >>> print(transcript.language) 'English' @@ -435,10 +448,10 @@ def __init__(self, http_client, video_id, url, language, language_code, is_gener def fetch(self, preserve_formatting=False): """Get that transcript data! 🎯 - + Args: preserve_formatting (bool): Keep HTML formatting? Default is nah fam. - + Returns: list: That sweet transcript data with text, start time, and duration! 📝 """ @@ -462,13 +475,13 @@ def is_translatable(self): def translate(self, language_code): """Translate to another language! 🌎 - + Args: language_code (str): Which language you want fam? - + Returns: Transcript: A fresh transcript in your requested language! 🔄 - + Raises: NotTranslatableError: If we can't translate this one 😢 TranslationLanguageNotAvailableError: If that language isn't available 🚫 @@ -492,13 +505,13 @@ def translate(self, language_code): class TranscriptParser: """Parsing those transcripts like a pro! 🎯 - + >>> parser = TranscriptParser(preserve_formatting=True) >>> data = parser.parse(xml_data) >>> print(data[0]) {'text': 'Never gonna give you up', 'start': 0.0, 'duration': 4.5} """ - + _FORMATTING_TAGS = [ 'strong', # For that extra emphasis 💪 'em', # When you need that italic swag 🎨 @@ -538,17 +551,17 @@ def parse(self, plain_data): for xml_element in ElementTree.fromstring(plain_data) if xml_element.text is not None ] - except ElementTree.ParseError as e: + except ElementTree.ParseError: # If XML parsing fails, try to extract text manually return self._fallback_parse(plain_data) - + def _fallback_parse(self, plain_data): """Fallback parsing method if XML parsing fails.""" results = [] # Try regex pattern matching pattern = r']*>([^<]*)' matches = re.findall(pattern, plain_data, re.DOTALL) - + for start, dur, text in matches: text = html.unescape(text) text = re.sub(self._html_regex, '', text) @@ -558,7 +571,7 @@ def _fallback_parse(self, plain_data): 'start': float(start), 'duration': float(dur), }) - + return results @@ -579,4 +592,4 @@ def _raise_http_errors(response, video_id): video_url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ" transcript = YTTranscriber.get_transcript(video_url, languages=None) print("Here's what we got! 🔥") - print(transcript[:5]) \ No newline at end of file + print(transcript[:5]) diff --git a/webscout/Extra/YTToolkit/ytapi/__init__.py b/webscout/Extra/YTToolkit/ytapi/__init__.py index 52b9160f..d965d840 100644 --- a/webscout/Extra/YTToolkit/ytapi/__init__.py +++ b/webscout/Extra/YTToolkit/ytapi/__init__.py @@ -1,13 +1,13 @@ +from .captions import Captions +from .channel import Channel from .errors import * -from .video import Video -from .query import Search from .extras import Extras -from .channel import Channel +from .hashtag import Hashtag from .playlist import Playlist -from .suggestions import Suggestions +from .query import Search from .shorts import Shorts -from .hashtag import Hashtag -from .captions import Captions +from .suggestions import Suggestions +from .video import Video __all__ = [ 'Video', diff --git a/webscout/Extra/YTToolkit/ytapi/captions.py b/webscout/Extra/YTToolkit/ytapi/captions.py index ecbb1ade..e8ed602f 100644 --- a/webscout/Extra/YTToolkit/ytapi/captions.py +++ b/webscout/Extra/YTToolkit/ytapi/captions.py @@ -3,69 +3,70 @@ This module wraps the YTTranscriber for a simplified interface. """ import re -from typing import List, Optional, Dict, Any +from typing import Any, Dict, List, Optional -# Use the existing robust YTTranscriber -from webscout.Extra.YTToolkit.transcriber import YTTranscriber, TranscriptListFetcher import requests +# Use the existing robust YTTranscriber +from webscout.Extra.YTToolkit.transcriber import TranscriptListFetcher, YTTranscriber + class Captions: """Class for YouTube captions and transcripts. - + Uses YTTranscriber internally for reliable transcript fetching. - + Example: >>> from webscout.Extra.YTToolkit.ytapi import Captions >>> transcript = Captions.get_transcript("dQw4w9WgXcQ") >>> print(transcript[:100]) """ - + @staticmethod def _extract_video_id(video_id: str) -> str: """Extract clean video ID from URL or ID.""" if not video_id: return "" - + patterns = [ r'(?:v=|youtu\.be/|shorts/)([a-zA-Z0-9_-]{11})', r'^([a-zA-Z0-9_-]{11})$' ] - + for pattern in patterns: match = re.search(pattern, video_id) if match: return match.group(1) - + return video_id - + @staticmethod def get_available_languages(video_id: str) -> List[Dict[str, str]]: """ Get available caption languages for a video. - + Args: video_id: YouTube video ID or URL - + Returns: List of dicts with 'code', 'name', 'is_auto' for each language """ if not video_id: return [] - + video_id = Captions._extract_video_id(video_id) - + try: session = requests.Session() session.headers.update({ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' }) - + fetcher = TranscriptListFetcher(session) transcript_list = fetcher.fetch(video_id) - + languages = [] - + # Add manually created transcripts for transcript in transcript_list._manually_created_transcripts.values(): languages.append({ @@ -73,7 +74,7 @@ def get_available_languages(video_id: str) -> List[Dict[str, str]]: 'name': transcript.language, 'is_auto': False }) - + # Add generated transcripts for transcript in transcript_list._generated_transcripts.values(): languages.append({ @@ -81,46 +82,46 @@ def get_available_languages(video_id: str) -> List[Dict[str, str]]: 'name': transcript.language, 'is_auto': True }) - + return languages except Exception: return [] - + @staticmethod def get_transcript(video_id: str, language: str = "en") -> Optional[str]: """ Get plain text transcript for a video. - + Args: video_id: YouTube video ID or URL language: Language code (e.g., 'en', 'es') - + Returns: Transcript text or None """ timed = Captions.get_timed_transcript(video_id, language) if not timed: return None - + return " ".join([entry['text'] for entry in timed]) - + @staticmethod def get_timed_transcript(video_id: str, language: str = "en") -> Optional[List[Dict[str, Any]]]: """ Get transcript with timestamps. - + Args: video_id: YouTube video ID or URL language: Language code (e.g., 'en', 'es'). Use 'any' for first available. - + Returns: List of dicts with 'text', 'start', 'duration' or None """ if not video_id: return None - + video_id = Captions._extract_video_id(video_id) - + try: # Use YTTranscriber for reliable fetching transcript = YTTranscriber.get_transcript( @@ -137,46 +138,46 @@ def get_timed_transcript(video_id: str, language: str = "en") -> Optional[List[D except Exception: pass return None - + @staticmethod def search_transcript(video_id: str, query: str, language: str = "en") -> List[Dict[str, Any]]: """ Search within a video's transcript. - + Args: video_id: YouTube video ID or URL query: Text to search for language: Language code - + Returns: List of matching segments with timestamps """ if not query: return [] - + transcript = Captions.get_timed_transcript(video_id, language) if not transcript: return [] - + query_lower = query.lower() results = [] - + for entry in transcript: if query_lower in entry['text'].lower(): results.append(entry) - + return results if __name__ == "__main__": # Test with a video that has captions video_id = "dQw4w9WgXcQ" - + print("Available languages:") langs = Captions.get_available_languages(video_id) for lang in langs[:5]: print(f" - {lang['code']}: {lang['name']} (auto: {lang.get('is_auto', False)})") - + print("\nGetting transcript:") transcript = Captions.get_timed_transcript(video_id) if transcript: diff --git a/webscout/Extra/YTToolkit/ytapi/channel.py b/webscout/Extra/YTToolkit/ytapi/channel.py index c53f05ea..a9dbcb6a 100644 --- a/webscout/Extra/YTToolkit/ytapi/channel.py +++ b/webscout/Extra/YTToolkit/ytapi/channel.py @@ -1,19 +1,13 @@ import json import re +from typing import Dict, List, Optional +from urllib.parse import unquote -from .https import ( - channel_about, - streams_data, - uploads_data, - channel_playlists, - upcoming_videos -) -from .video import Video +from .https import channel_about, channel_playlists, streams_data, upcoming_videos, uploads_data +from .patterns import _ChannelPatterns as Patterns from .pool import collect from .utils import dup_filter -from urllib.parse import unquote -from typing import List, Optional, Dict -from .patterns import _ChannelPatterns as Patterns +from .video import Video class Channel: @@ -87,11 +81,11 @@ def __prepare_metadata(self) -> Optional[Dict[str, any]]: ] extracted = collect(lambda x: x.findall(self._about_page) or None, patterns) name, avatar, banner, verified, socials = [e[0] if e else None for e in extracted] - + # Add robust error handling for info extraction info_pattern = re.compile("\\[{\"aboutChannelRenderer\":(.*?)],") info_match = info_pattern.search(self._about_page) - + if not info_match: # Fallback metadata for search results or incomplete channel data return { @@ -110,11 +104,11 @@ def __prepare_metadata(self) -> Optional[Dict[str, any]]: "verified": bool(verified), "socials": unquote(socials) if socials is not None else None } - + try: info_str = info_match.group(1) + "]}}}}" info = json.loads(info_str)["metadata"]["aboutChannelViewModel"] - + return { "id": info.get("channelId", self._usable_id), "name": name, @@ -232,7 +226,7 @@ def last_streamed(self) -> Optional[str]: """ ids = self.old_streams return ids[0] if ids else None - + def uploads(self, limit: int = 20) -> Optional[List[str]]: """ Fetches the ids of all uploaded videos diff --git a/webscout/Extra/YTToolkit/ytapi/extras.py b/webscout/Extra/YTToolkit/ytapi/extras.py index 79339d51..a739c93d 100644 --- a/webscout/Extra/YTToolkit/ytapi/extras.py +++ b/webscout/Extra/YTToolkit/ytapi/extras.py @@ -1,15 +1,16 @@ +from typing import List, Optional + from .https import ( - trending_videos, - trending_songs, - trending_games, + _get_trending_learning_videos, trending_feeds, + trending_games, + trending_songs, + trending_sports, trending_streams, - _get_trending_learning_videos, - trending_sports + trending_videos, ) -from .utils import dup_filter, request from .patterns import _ExtraPatterns as Patterns -from typing import Optional, List +from .utils import dup_filter, request class Extras: @@ -18,12 +19,12 @@ class Extras: def trending_videos(limit: int = None) -> Optional[List[str]]: """ Get trending videos from YouTube. - + Args: - limit (int, optional): Maximum number of videos to return. - Alternatively, manual slicing can be used: + limit (int, optional): Maximum number of videos to return. + Alternatively, manual slicing can be used: Extras.trending_videos()[:5] - + Returns: Optional[List[str]]: List of video IDs or None if no videos found """ @@ -34,12 +35,12 @@ def trending_videos(limit: int = None) -> Optional[List[str]]: def music_videos(limit: int = None) -> Optional[List[str]]: """ Get trending music videos from YouTube. - + Args: limit (int, optional): Maximum number of videos to return. - Alternatively, manual slicing can be used: + Alternatively, manual slicing can be used: Extras.music_videos()[:5] - + Returns: Optional[List[str]]: List of video IDs or None if no videos found """ @@ -50,27 +51,27 @@ def music_videos(limit: int = None) -> Optional[List[str]]: def gaming_videos(limit: int = None) -> Optional[List[str]]: """ Get trending gaming videos from YouTube. - + Args: limit (int, optional): Maximum number of videos to return. - Alternatively, manual slicing can be used: + Alternatively, manual slicing can be used: Extras.gaming_videos()[:5] - + Returns: Optional[List[str]]: List of video IDs or None if no videos found """ return dup_filter(Patterns.video_id.findall(trending_games()), limit) - + @staticmethod def news_videos(limit: int = None) -> Optional[List[str]]: """ Get trending news videos from YouTube. - + Args: limit (int, optional): Maximum number of videos to return. - Alternatively, manual slicing can be used: + Alternatively, manual slicing can be used: Extras.news_videos()[:5] - + Returns: Optional[List[str]]: List of video IDs or None if no videos found """ @@ -80,12 +81,12 @@ def news_videos(limit: int = None) -> Optional[List[str]]: def live_videos(limit: int = None) -> Optional[List[str]]: """ Get trending live videos from YouTube. - + Args: limit (int, optional): Maximum number of videos to return. - Alternatively, manual slicing can be used: + Alternatively, manual slicing can be used: Extras.live_videos()[:5] - + Returns: Optional[List[str]]: List of video IDs or None if no videos found """ @@ -95,12 +96,12 @@ def live_videos(limit: int = None) -> Optional[List[str]]: def educational_videos(limit: int = None) -> Optional[List[str]]: """ Get trending educational videos from YouTube. - + Args: limit (int, optional): Maximum number of videos to return. - Alternatively, manual slicing can be used: + Alternatively, manual slicing can be used: Extras.educational_videos()[:5] - + Returns: Optional[List[str]]: List of video IDs or None if no videos found """ @@ -110,12 +111,12 @@ def educational_videos(limit: int = None) -> Optional[List[str]]: def sport_videos(limit: int = None) -> Optional[List[str]]: """ Get trending sports videos from YouTube. - + Args: limit (int, optional): Maximum number of videos to return. - Alternatively, manual slicing can be used: + Alternatively, manual slicing can be used: Extras.sport_videos()[:5] - + Returns: Optional[List[str]]: List of video IDs or None if no videos found """ @@ -125,10 +126,10 @@ def sport_videos(limit: int = None) -> Optional[List[str]]: def shorts_videos(limit: int = None) -> Optional[List[str]]: """ Get trending YouTube Shorts. - + Args: limit (int, optional): Maximum number of Shorts to return. - + Returns: Optional[List[str]]: List of video IDs or None if no Shorts found """ @@ -143,10 +144,10 @@ def shorts_videos(limit: int = None) -> Optional[List[str]]: def movies(limit: int = None) -> Optional[List[str]]: """ Get featured movies from YouTube. - + Args: limit (int, optional): Maximum number of movies to return. - + Returns: Optional[List[str]]: List of video IDs or None if no movies found """ @@ -161,10 +162,10 @@ def movies(limit: int = None) -> Optional[List[str]]: def podcasts(limit: int = None) -> Optional[List[str]]: """ Get trending podcasts from YouTube. - + Args: limit (int, optional): Maximum number of podcasts to return. - + Returns: Optional[List[str]]: List of video IDs or None if no podcasts found """ diff --git a/webscout/Extra/YTToolkit/ytapi/hashtag.py b/webscout/Extra/YTToolkit/ytapi/hashtag.py index 1da9df95..c57d289d 100644 --- a/webscout/Extra/YTToolkit/ytapi/hashtag.py +++ b/webscout/Extra/YTToolkit/ytapi/hashtag.py @@ -1,79 +1,80 @@ """YouTube Hashtag functionality.""" import re -from typing import List, Optional, Dict, Any +from typing import Any, Dict, List from urllib.parse import quote -from .utils import request, dup_filter + from .patterns import _ExtraPatterns as Patterns +from .utils import dup_filter, request class Hashtag: """Class for YouTube hashtag operations.""" - + BASE_URL = "https://www.youtube.com/hashtag" - + @staticmethod def get_videos(tag: str, limit: int = 20) -> List[str]: """ Get videos associated with a hashtag. - + Args: tag: Hashtag (with or without #) limit: Maximum number of videos to return - + Returns: List of video IDs """ if not tag: return [] - + # Remove # if present and clean the tag tag = tag.lstrip('#').strip().lower() tag = re.sub(r'[^a-zA-Z0-9]', '', tag) - + if not tag: return [] - + url = f"{Hashtag.BASE_URL}/{quote(tag)}" - + try: html = request(url) video_ids = Patterns.video_id.findall(html) return dup_filter(video_ids, limit) except Exception: return [] - + @staticmethod def get_metadata(tag: str) -> Dict[str, Any]: """ Get metadata about a hashtag. - + Args: tag: Hashtag (with or without #) - + Returns: Dictionary with hashtag info (name, video_count if available) """ if not tag: return {} - + tag = tag.lstrip('#').strip().lower() tag = re.sub(r'[^a-zA-Z0-9]', '', tag) - + if not tag: return {} - + url = f"{Hashtag.BASE_URL}/{quote(tag)}" - + try: html = request(url) - + # Try to extract video count if available video_count_match = re.search(r'"videoCountText":\s*\{\s*"runs":\s*\[\s*\{\s*"text":\s*"([^"]+)"', html) video_count = video_count_match.group(1) if video_count_match else None - + # Get sample of videos video_ids = Patterns.video_id.findall(html) - + return { 'tag': tag, 'url': url, @@ -82,21 +83,21 @@ def get_metadata(tag: str) -> Dict[str, Any]: } except Exception: return {'tag': tag, 'url': url} - + @staticmethod def extract_from_text(text: str) -> List[str]: """ Extract hashtags from text. - + Args: text: Text containing hashtags - + Returns: List of hashtags found """ if not text: return [] - + pattern = r'#([a-zA-Z0-9_]+)' matches = re.findall(pattern, text) return list(dict.fromkeys(matches)) # Remove duplicates, preserve order @@ -107,12 +108,12 @@ def extract_from_text(text: str) -> List[str]: videos = Hashtag.get_videos("python", 5) for vid in videos: print(f" - {vid}") - + print("\nHashtag metadata:") meta = Hashtag.get_metadata("coding") print(f" Tag: {meta.get('tag')}") print(f" Videos: {len(meta.get('sample_videos', []))}") - + print("\nExtract hashtags:") text = "Check out my new #python #tutorial for #beginners!" tags = Hashtag.extract_from_text(text) diff --git a/webscout/Extra/YTToolkit/ytapi/https.py b/webscout/Extra/YTToolkit/ytapi/https.py index 5f8c66db..46e7cfa2 100644 --- a/webscout/Extra/YTToolkit/ytapi/https.py +++ b/webscout/Extra/YTToolkit/ytapi/https.py @@ -1,4 +1,5 @@ from urllib.parse import quote + from .utils import request diff --git a/webscout/Extra/YTToolkit/ytapi/playlist.py b/webscout/Extra/YTToolkit/ytapi/playlist.py index 3143790c..812ae3f0 100644 --- a/webscout/Extra/YTToolkit/ytapi/playlist.py +++ b/webscout/Extra/YTToolkit/ytapi/playlist.py @@ -1,10 +1,10 @@ import re -from typing import Dict, Any +from typing import Any, Dict -from .pool import collect -from .utils import dup_filter from .https import playlist_data from .patterns import _PlaylistPatterns as Patterns +from .pool import collect +from .utils import dup_filter class Playlist: @@ -56,4 +56,4 @@ def metadata(self) -> Dict[str, Any]: 'video_count': data[1] if data else None, 'thumbnail': data[2] if data else None, 'videos': dup_filter(ext[3]) - } \ No newline at end of file + } diff --git a/webscout/Extra/YTToolkit/ytapi/pool.py b/webscout/Extra/YTToolkit/ytapi/pool.py index 50120ccf..f78c6abb 100644 --- a/webscout/Extra/YTToolkit/ytapi/pool.py +++ b/webscout/Extra/YTToolkit/ytapi/pool.py @@ -1,8 +1,8 @@ import concurrent.futures -from typing import Callable, List, Any +from typing import Any, Callable, List def collect(func: Callable, args: List[Any]) -> List[Any]: max_workers = len(args) or 1 with concurrent.futures.ThreadPoolExecutor(max_workers) as exe: - return list(exe.map(func, args)) \ No newline at end of file + return list(exe.map(func, args)) diff --git a/webscout/Extra/YTToolkit/ytapi/query.py b/webscout/Extra/YTToolkit/ytapi/query.py index eef73637..7fc6132f 100644 --- a/webscout/Extra/YTToolkit/ytapi/query.py +++ b/webscout/Extra/YTToolkit/ytapi/query.py @@ -1,12 +1,12 @@ -from .utils import dup_filter -from .video import Video +from typing import List, Optional +from urllib.parse import quote + from .channel import Channel -from .playlist import Playlist +from .https import find_channels, find_playlists, find_videos from .patterns import _QueryPatterns as Patterns -from typing import Optional, Dict, Any, List -from .https import find_videos, find_channels, find_playlists -from urllib.parse import quote -from .utils import request +from .playlist import Playlist +from .utils import dup_filter, request +from .video import Video class Search: @@ -42,11 +42,11 @@ def playlists(keywords: str, limit: int = 20) -> Optional[List[str]]: def shorts(keywords: str, limit: int = 20) -> Optional[List[str]]: """ Search for YouTube Shorts. - + Args: keywords: Search query limit: Maximum number of results - + Returns: List of video IDs for matching Shorts """ @@ -62,11 +62,11 @@ def shorts(keywords: str, limit: int = 20) -> Optional[List[str]]: def live_streams(keywords: str, limit: int = 20) -> Optional[List[str]]: """ Search for live streams. - + Args: keywords: Search query limit: Maximum number of results - + Returns: List of video IDs for live streams """ @@ -82,12 +82,12 @@ def live_streams(keywords: str, limit: int = 20) -> Optional[List[str]]: def videos_by_duration(keywords: str, duration: str = "short", limit: int = 20) -> Optional[List[str]]: """ Search videos filtered by duration. - + Args: keywords: Search query duration: Duration filter - "short" (<4 min), "medium" (4-20 min), "long" (>20 min) limit: Maximum number of results - + Returns: List of video IDs """ @@ -108,12 +108,12 @@ def videos_by_duration(keywords: str, duration: str = "short", limit: int = 20) def videos_by_upload_date(keywords: str, when: str = "today", limit: int = 20) -> Optional[List[str]]: """ Search videos filtered by upload date. - + Args: keywords: Search query when: Time filter - "hour", "today", "week", "month", "year" limit: Maximum number of results - + Returns: List of video IDs """ @@ -137,7 +137,7 @@ def videos_by_upload_date(keywords: str, when: str = "today", limit: int = 20) - print("Testing Search.shorts:") shorts = Search.shorts("funny", 5) print(f" Found: {shorts}") - + print("\nTesting Search.live_streams:") live = Search.live_streams("music", 5) print(f" Found: {live}") diff --git a/webscout/Extra/YTToolkit/ytapi/shorts.py b/webscout/Extra/YTToolkit/ytapi/shorts.py index a34789d5..eaf5ae0e 100644 --- a/webscout/Extra/YTToolkit/ytapi/shorts.py +++ b/webscout/Extra/YTToolkit/ytapi/shorts.py @@ -1,10 +1,11 @@ """YouTube Shorts functionality.""" import re -from typing import List, Optional, Dict, Any -from urllib.request import Request, urlopen +from typing import List from urllib.error import HTTPError -from .utils import request, dup_filter +from urllib.request import Request, urlopen + from .patterns import _ExtraPatterns as Patterns +from .utils import dup_filter, request try: from webscout.litagent.agent import LitAgent @@ -15,31 +16,31 @@ class Shorts: """Class for YouTube Shorts operations.""" - + SHORTS_URL = "https://www.youtube.com/shorts" - + @staticmethod def is_short(video_id: str) -> bool: """ Check if a video is a YouTube Short. - + Args: video_id: YouTube video ID - + Returns: True if video is a Short, False otherwise """ if not video_id: return False - + # Clean video ID if "youtube.com" in video_id or "youtu.be" in video_id: match = re.search(r'(?:v=|shorts/|youtu\.be/)([a-zA-Z0-9_-]{11})', video_id) if match: video_id = match.group(1) - + url = f"https://www.youtube.com/shorts/{video_id}" - + try: headers = { "User-Agent": _USER_AGENT, @@ -57,15 +58,15 @@ def is_short(video_id: str) -> bool: return False except Exception: return False - + @staticmethod def get_trending(limit: int = 20) -> List[str]: """ Get trending YouTube Shorts. - + Args: limit: Maximum number of Shorts to return - + Returns: List of video IDs for trending Shorts """ @@ -77,26 +78,26 @@ def get_trending(limit: int = 20) -> List[str]: return dup_filter(video_ids, limit) except Exception: return [] - + @staticmethod def search(query: str, limit: int = 20) -> List[str]: """ Search for YouTube Shorts. - + Args: query: Search query limit: Maximum number of results - + Returns: List of video IDs for matching Shorts """ if not query: return [] - + from urllib.parse import quote # sp=EgIYAQ%253D%253D is the filter for Shorts url = f"https://www.youtube.com/results?search_query={quote(query)}&sp=EgIYAQ%253D%253D" - + try: html = request(url) video_ids = Patterns.video_id.findall(html) @@ -109,12 +110,12 @@ def search(query: str, limit: int = 20) -> List[str]: print("Testing Shorts.is_short:") # Test with a known Short ID (you'd replace with an actual Short ID) print(f" is_short test: {Shorts.is_short('abc123')}") - + print("\nTrending Shorts:") trending = Shorts.get_trending(5) for vid in trending: print(f" - {vid}") - + print("\nSearch Shorts:") results = Shorts.search("funny cats", 5) for vid in results: diff --git a/webscout/Extra/YTToolkit/ytapi/stream.py b/webscout/Extra/YTToolkit/ytapi/stream.py index 4727cc17..58474bff 100644 --- a/webscout/Extra/YTToolkit/ytapi/stream.py +++ b/webscout/Extra/YTToolkit/ytapi/stream.py @@ -1,9 +1,9 @@ import re -from typing import Dict, Any +from typing import Any, Dict -from .pool import collect from .https import video_data from .patterns import _VideoPatterns as Patterns +from .pool import collect class Video: @@ -60,4 +60,4 @@ def metadata(self) -> Dict[str, Any]: } if __name__ == '__main__': - print(Video('https://www.youtube.com/watch?v=9bZkp7q19f0').metadata) \ No newline at end of file + print(Video('https://www.youtube.com/watch?v=9bZkp7q19f0').metadata) diff --git a/webscout/Extra/YTToolkit/ytapi/suggestions.py b/webscout/Extra/YTToolkit/ytapi/suggestions.py index 54db6839..3fa8cd3a 100644 --- a/webscout/Extra/YTToolkit/ytapi/suggestions.py +++ b/webscout/Extra/YTToolkit/ytapi/suggestions.py @@ -1,8 +1,8 @@ """YouTube search suggestions and autocomplete.""" import json -from typing import List, Optional -from urllib.request import Request, urlopen +from typing import List from urllib.parse import quote +from urllib.request import Request, urlopen try: from webscout.litagent.agent import LitAgent @@ -13,36 +13,36 @@ class Suggestions: """Class for YouTube search suggestions and autocomplete.""" - + AUTOCOMPLETE_URL = "https://suggestqueries.google.com/complete/search" - + @staticmethod def autocomplete(query: str, language: str = "en") -> List[str]: """ Get YouTube autocomplete suggestions for a search query. - + Args: query: Search query to get suggestions for language: Language code (e.g., 'en', 'es', 'fr') - + Returns: List of autocomplete suggestions """ if not query: return [] - + url = f"{Suggestions.AUTOCOMPLETE_URL}?client=youtube&ds=yt&q={quote(query)}&hl={language}" - + headers = { "User-Agent": _USER_AGENT, "Accept": "application/json" } - + try: req = Request(url, headers=headers) response = urlopen(req, timeout=10) data = response.read().decode('utf-8') - + # Response is JSONP, extract JSON part # Format: window.google.ac.h(["query",[["suggestion1"],["suggestion2"],...]]) start = data.find('(') @@ -55,16 +55,16 @@ def autocomplete(query: str, language: str = "en") -> List[str]: return [] except Exception: return [] - + @staticmethod def trending_searches(language: str = "en", country: str = "US") -> List[str]: """ Get trending YouTube searches. - + Args: language: Language code country: Country code - + Returns: List of trending search terms """ @@ -73,7 +73,7 @@ def trending_searches(language: str = "en", country: str = "US") -> List[str]: for seed in ["", "how to", "what is", "best"]: suggestions = Suggestions.autocomplete(seed, language) trending.extend(suggestions[:3]) - + # Remove duplicates while preserving order seen = set() unique = [] @@ -81,7 +81,7 @@ def trending_searches(language: str = "en", country: str = "US") -> List[str]: if item not in seen: seen.add(item) unique.append(item) - + return unique[:20] @@ -90,7 +90,7 @@ def trending_searches(language: str = "en", country: str = "US") -> List[str]: suggestions = Suggestions.autocomplete("python tutorial") for s in suggestions[:5]: print(f" - {s}") - + print("\nTrending searches:") trending = Suggestions.trending_searches() for t in trending[:5]: diff --git a/webscout/Extra/YTToolkit/ytapi/utils.py b/webscout/Extra/YTToolkit/ytapi/utils.py index dda014ad..99d91aec 100644 --- a/webscout/Extra/YTToolkit/ytapi/utils.py +++ b/webscout/Extra/YTToolkit/ytapi/utils.py @@ -1,9 +1,10 @@ -from urllib.request import Request, urlopen from collections import OrderedDict from urllib.error import HTTPError -from .errors import TooManyRequests, InvalidURL, RequestError +from urllib.request import Request, urlopen + from webscout.litagent import LitAgent +from .errors import InvalidURL, RequestError, TooManyRequests __all__ = ['dup_filter', 'request'] @@ -14,16 +15,16 @@ def request(url: str, retry_attempts: int = 3) -> str: """ Send a request with a random user agent and built-in retry mechanism. - + Args: url (str): The URL to request retry_attempts (int, optional): Number of retry attempts. Defaults to 3. - + Raises: InvalidURL: If the URL cannot be found TooManyRequests: If rate-limited RequestError: For other request-related errors - + Returns: str: Decoded response content """ @@ -32,20 +33,20 @@ def request(url: str, retry_attempts: int = 3) -> str: headers = { "User-Agent": _USER_AGENT_GENERATOR.random() } - + req = Request(url, headers=headers) response = urlopen(req) return response.read().decode('utf-8') - + except HTTPError as e: if e.code == 404: raise InvalidURL(f'Cannot find anything with the requested URL: {url}') if e.code == 429: raise TooManyRequests(f'Rate-limited on attempt {attempt + 1}') - + if attempt == retry_attempts - 1: raise RequestError(f'HTTP Error {e.code}: {e.reason}') from e - + except Exception as e: if attempt == retry_attempts - 1: raise RequestError(f'Request failed: {e!r}') from None diff --git a/webscout/Extra/YTToolkit/ytapi/video.py b/webscout/Extra/YTToolkit/ytapi/video.py index 030b52c9..42642e93 100644 --- a/webscout/Extra/YTToolkit/ytapi/video.py +++ b/webscout/Extra/YTToolkit/ytapi/video.py @@ -1,8 +1,8 @@ -import re import json -from typing import Dict, Any, List, Optional, Generator +import re +from typing import Any, Dict, Generator, List, Optional + from .https import video_data -from urllib.request import Request, urlopen try: from webscout.litagent.agent import LitAgent @@ -282,12 +282,12 @@ def get_related_videos(self, limit: int = 10) -> List[str]: # Find related videos in the page data pattern = r'"watchNextEndScreenRenderer".*?"videoId":"([a-zA-Z0-9_-]{11})"' matches = re.findall(pattern, self._video_data) - + if not matches: # Fallback pattern pattern = r'"compactVideoRenderer".*?"videoId":"([a-zA-Z0-9_-]{11})"' matches = re.findall(pattern, self._video_data) - + # Remove duplicates and self seen = set() unique = [] @@ -297,7 +297,7 @@ def get_related_videos(self, limit: int = 10) -> List[str]: unique.append(vid) if len(unique) >= limit: break - + return unique def get_chapters(self) -> Optional[List[Dict[str, Any]]]: @@ -310,7 +310,7 @@ def get_chapters(self) -> Optional[List[Dict[str, Any]]]: # Look for chapter data in the page pattern = r'"chapterRenderer":\s*\{[^}]*"title":\s*\{\s*"simpleText":\s*"([^"]+)"[^}]*"timeRangeStartMillis":\s*(\d+)' matches = re.findall(pattern, self._video_data) - + if not matches: # Alternative pattern pattern = r'"chapters":\s*\[(.*?)\]' @@ -321,10 +321,10 @@ def get_chapters(self) -> Optional[List[Dict[str, Any]]]: chapters_str = '[' + chapter_match.group(1) + ']' chapters_data = json.loads(chapters_str) return chapters_data - except: + except Exception: pass return None - + chapters = [] for title, start_ms in matches: chapters.append({ @@ -332,7 +332,7 @@ def get_chapters(self) -> Optional[List[Dict[str, Any]]]: 'start_seconds': int(start_ms) / 1000, 'start_time': f"{int(int(start_ms)/1000//60)}:{int(int(start_ms)/1000%60):02d}" }) - + return chapters if chapters else None def stream_comments(self, limit: int = 20) -> Generator[Dict[str, Any], None, None]: @@ -358,10 +358,10 @@ def stream_comments(self, limit: int = 20) -> Generator[Dict[str, Any], None, No # Simpler pattern r'"authorText":"([^"]+)".*?"contentText":"([^"]*)"', ] - + count = 0 seen_comments = set() - + for pattern in patterns: if count >= limit: break @@ -374,13 +374,13 @@ def stream_comments(self, limit: int = 20) -> Generator[Dict[str, Any], None, No if comment_key in seen_comments: continue seen_comments.add(comment_key) - + # Clean up text text = text.replace('\\n', '\n') text = text.replace('\\u0026', '&') text = text.replace('\\u003c', '<') text = text.replace('\\u003e', '>') - + yield { 'author': author, 'text': text, @@ -396,7 +396,7 @@ def stream_comments(self, limit: int = 20) -> Generator[Dict[str, Any], None, No print(f"Is Short: {video.is_short}") print(f"Hashtags: {video.hashtags}") print(f"Related videos: {video.get_related_videos(5)}") - + chapters = video.get_chapters() if chapters: print(f"Chapters: {chapters[:3]}") diff --git a/webscout/Extra/__init__.py b/webscout/Extra/__init__.py index 42b3e13d..674f1267 100644 --- a/webscout/Extra/__init__.py +++ b/webscout/Extra/__init__.py @@ -1,6 +1,6 @@ from .gguf import * +from .GitToolkit import * +from .tempmail import * from .weather import * from .weather_ascii import * from .YTToolkit import * -from .GitToolkit import * -from .tempmail import * \ No newline at end of file diff --git a/webscout/Extra/gguf.py b/webscout/Extra/gguf.py index fab2ebe8..a352b830 100644 --- a/webscout/Extra/gguf.py +++ b/webscout/Extra/gguf.py @@ -1,18 +1,19 @@ +import os +import platform +import signal import subprocess -import os import sys -import signal import tempfile -import platform from pathlib import Path -from typing import Optional, Dict, List, Any, Union, Literal, TypedDict, Set +from typing import Dict, List, Optional, Set, TypedDict from huggingface_hub import HfApi -from webscout.zeroart import figlet_format from rich.console import Console from rich.panel import Panel from rich.table import Table + from webscout.swiftcli import CLI, option +from webscout.zeroart import figlet_format console = Console() @@ -26,7 +27,7 @@ class QuantizationMethod(TypedDict): class ModelConverter: """Handles the conversion of Hugging Face models to GGUF format.""" - + VALID_METHODS: Dict[str, str] = { # Full precision types "f32": "32-bit floating point - full precision, largest size", @@ -56,7 +57,7 @@ class ModelConverter: "tq1_0": "1-bit ternary quantization - experimental, extreme compression", "tq2_0": "2-bit ternary quantization - experimental, very small size" } - + VALID_IMATRIX_METHODS: Dict[str, str] = { # 1-bit IQ types (extreme compression, requires imatrix) "iq1_s": "1-bit IQ small - extreme compression, requires imatrix", @@ -88,7 +89,7 @@ class ModelConverter: } # Default output type options (matches llama.cpp) VALID_OUTTYPES: Set[str] = {"f32", "f16", "bf16", "q8_0", "tq1_0", "tq2_0", "auto"} - + def __init__( self, model_id: str, @@ -129,12 +130,12 @@ def __init__( self.small_first_shard = small_first_shard # Determine if we only need the base conversion (no quantization) self.base_only = self.outtype in self.VALID_OUTTYPES and len(self.quantization_methods) == 1 and self.quantization_methods[0] in ["fp16", "f16", "f32", "bf16", "auto"] - + def validate_inputs(self) -> None: """Validates all input parameters.""" - if not '/' in self.model_id: + if '/' not in self.model_id: raise ValueError("Invalid model ID format. Expected format: 'organization/model-name'") - + if self.use_imatrix: invalid_methods = [m for m in self.quantization_methods if m not in self.VALID_IMATRIX_METHODS] if invalid_methods: @@ -151,32 +152,32 @@ def validate_inputs(self) -> None: f"Invalid quantization methods: {', '.join(invalid_methods)}.\n" f"Valid methods are: {', '.join(self.VALID_METHODS.keys())}" ) - + if bool(self.username) != bool(self.token): raise ValueError("Both username and token must be provided for upload, or neither.") - + # Validate outtype if self.outtype not in self.VALID_OUTTYPES: raise ValueError( f"Invalid output type: {self.outtype}.\n" f"Valid types are: {', '.join(self.VALID_OUTTYPES)}" ) - + if self.split_model and self.split_max_size: try: # Support K, M, G units (like llama.cpp's split_str_to_n_bytes) if self.split_max_size[-1].upper() in ['K', 'M', 'G']: - size = int(self.split_max_size[:-1]) + int(self.split_max_size[:-1]) unit = self.split_max_size[-1].upper() if unit not in ['K', 'M', 'G']: raise ValueError("Split max size must end with K, M, or G") elif self.split_max_size.isnumeric(): - size = int(self.split_max_size) + int(self.split_max_size) else: raise ValueError("Invalid format") except (ValueError, IndexError): raise ValueError("Invalid split max size format. Use format like '256M', '5G', or numeric bytes") - + @staticmethod def check_dependencies() -> Dict[str, bool]: """Check if all required dependencies are installed with cross-platform support.""" @@ -221,7 +222,7 @@ def check_dependencies() -> Dict[str, bool]: if result.returncode == 0: status['python'] = True break - except: + except Exception: continue # Check for C++ compiler @@ -236,14 +237,14 @@ def check_dependencies() -> Dict[str, bool]: if result.returncode == 0: status['cpp_compiler'] = True break - except: + except Exception: continue dependencies['python'] = 'Python interpreter' dependencies['cpp_compiler'] = 'C++ compiler (g++, clang++, or MSVC)' return status - + def detect_hardware(self) -> Dict[str, bool]: """Detect available hardware acceleration with improved cross-platform support.""" hardware: Dict[str, bool] = { @@ -340,7 +341,7 @@ def detect_hardware(self) -> Dict[str, bool]: # Check for BLAS libraries try: - import numpy as np # type: ignore + import numpy as np # type: ignore # Check if numpy is linked with optimized BLAS config = np.__config__.show() if any(lib in str(config).lower() for lib in ['openblas', 'mkl', 'atlas', 'blis']): @@ -358,7 +359,7 @@ def detect_hardware(self) -> Dict[str, bool]: hardware['blas'] = True return hardware - + def setup_llama_cpp(self) -> None: """Sets up and builds llama.cpp repository with robust error handling.""" llama_path = self.workspace / "llama.cpp" @@ -401,12 +402,12 @@ def setup_llama_cpp(self) -> None: # In Nix, we need to use the system Python packages try: # Try to import required packages to check if they're available - import torch # type: ignore - import numpy # type: ignore - import sentencepiece # type: ignore - import transformers # type: ignore + import numpy # type: ignore + import sentencepiece # type: ignore + import torch # type: ignore + import transformers # type: ignore console.print("[green]Required Python packages are already installed.") - except ImportError as e: + except ImportError: console.print("[red]Missing required Python packages in Nix environment.") console.print("[yellow]Please install them using:") console.print("nix-shell -p python3Packages.torch python3Packages.numpy python3Packages.sentencepiece python3Packages.transformers") @@ -515,7 +516,7 @@ def setup_llama_cpp(self) -> None: cmake_args.extend(['-G', 'Visual Studio 17 2022']) else: cmake_args.extend(['-G', 'MinGW Makefiles']) - except: + except Exception: cmake_args.extend(['-G', 'MinGW Makefiles']) else: # Use Ninja if available on Unix systems @@ -523,7 +524,7 @@ def setup_llama_cpp(self) -> None: ninja_cmd = 'ninja' if system != 'Windows' else 'ninja.exe' if subprocess.run(['which', ninja_cmd], capture_output=True).returncode == 0: cmake_args.extend(['-G', 'Ninja']) - except: + except Exception: pass # Fall back to default generator # Configure the build with error handling and multiple fallback strategies @@ -627,24 +628,24 @@ def setup_llama_cpp(self) -> None: finally: os.chdir(original_cwd) - + def display_config(self) -> None: """Displays the current configuration in a formatted table.""" table = Table(title="Configuration", show_header=True, header_style="bold magenta") table.add_column("Setting", style="cyan") table.add_column("Value", style="green") - + table.add_row("Model ID", self.model_id) table.add_row("Model Name", self.model_name) table.add_row("Username", self.username or "Not provided") table.add_row("Token", "****" if self.token else "Not provided") table.add_row("Quantization Methods", "\n".join( - f"{method} ({self.VALID_METHODS[method]})" + f"{method} ({self.VALID_METHODS[method]})" for method in self.quantization_methods )) - + console.print(Panel(table)) - + def get_binary_path(self, binary_name: str) -> str: """Get the correct path to llama.cpp binaries based on platform.""" system = platform.system() @@ -725,7 +726,7 @@ def generate_importance_matrix(self, model_path: str, train_data_path: str, outp raise ConversionError(f"Could not execute llama-imatrix binary: {imatrix_binary}") console.print("[green]Importance matrix generation completed.") - + def split_model(self, model_path: str, outdir: str) -> List[str]: """Splits the model into smaller chunks with improved error handling.""" split_binary = self.get_binary_path("llama-gguf-split") @@ -774,7 +775,7 @@ def split_model(self, model_path: str, outdir: str) -> List[str]: console.print(f"[green]Found {len(split_files)} split files: {', '.join(split_files)}") return split_files - + def upload_split_files(self, split_files: List[str], outdir: str, repo_id: str) -> None: """Uploads split model files to Hugging Face.""" api = HfApi(token=self.token) @@ -792,7 +793,7 @@ def upload_split_files(self, split_files: List[str], outdir: str, repo_id: str) except Exception as e: console.print(f"[red]✗ Failed to upload {file}: {e}") raise ConversionError(f"Error uploading file {file}: {e}") - + def generate_readme(self, quantized_files: List[str]) -> str: """Generate a README.md file for the Hugging Face Hub.""" readme = f"""# {self.model_name} GGUF @@ -910,22 +911,22 @@ def convert(self) -> None: # Display banner and configuration console.print(f"[bold green]{figlet_format('GGUF Converter')}") self.display_config() - + # Validate inputs self.validate_inputs() - + # Check dependencies deps = self.check_dependencies() missing = [name for name, installed in deps.items() if not installed and name != 'ninja'] if missing: raise ConversionError(f"Missing required dependencies: {', '.join(missing)}") - + # Setup llama.cpp self.setup_llama_cpp() - + # Determine if we need temporary directories (only for uploads) needs_temp = bool(self.username and self.token) - + if needs_temp: # Use temporary directories for upload case with tempfile.TemporaryDirectory() as outdir: @@ -942,7 +943,7 @@ def convert(self) -> None: # Clean up temporary download directory import shutil shutil.rmtree(tmpdir, ignore_errors=True) - + # Display success message console.print(Panel.fit( "[bold green]✓[/] Conversion completed successfully!\n\n" @@ -950,7 +951,7 @@ def convert(self) -> None: title="Success", border_style="green" )) - + except Exception as e: console.print(Panel.fit( f"[bold red]✗[/] {str(e)}", @@ -958,13 +959,13 @@ def convert(self) -> None: border_style="red" )) raise - + def _convert_with_dirs(self, tmpdir: str, outdir: str) -> None: """Helper method to perform conversion with given directories.""" # Use outtype for base filename (e.g., model.f16.gguf, model.bf16.gguf) outtype_suffix = self.outtype if self.outtype != "auto" else "f16" base_gguf = str(Path(outdir)/f"{self.model_name}.{outtype_suffix}.gguf") - + # Download model (or use remote mode) local_dir = Path(tmpdir)/self.model_name if self.remote: @@ -984,7 +985,7 @@ def _convert_with_dirs(self, tmpdir: str, outdir: str) -> None: local_dir=local_dir, local_dir_use_symlinks=False ) - + # Convert to GGUF with specified outtype console.print(f"[bold green]Converting to {self.outtype}...") @@ -1014,7 +1015,7 @@ def _convert_with_dirs(self, tmpdir: str, outdir: str) -> None: "--outtype", self.outtype, "--outfile", base_gguf ] - + # Add optional flags based on new llama.cpp features if self.vocab_only: convert_cmd.append("--vocab-only") @@ -1048,7 +1049,7 @@ def _convert_with_dirs(self, tmpdir: str, outdir: str) -> None: raise ConversionError(f"Conversion completed but output file not found: {base_gguf}") console.print(f"[green]Model converted to {self.outtype} successfully!") - + # If base_only is True, we're done after base conversion (no quantization needed) if self.base_only: gguf_filename = f"{self.model_name}.{outtype_suffix}.gguf" @@ -1076,14 +1077,14 @@ def _convert_with_dirs(self, tmpdir: str, outdir: str) -> None: console.print(f"[red]✗ Failed to upload {gguf_filename}: {e}") raise ConversionError(f"Error uploading model file: {e}") return - + # Generate importance matrix if needed imatrix_path: Optional[str] = None if self.use_imatrix: train_data_path = self.train_data_file if self.train_data_file else "llama.cpp/groups_merged.txt" imatrix_path = str(Path(outdir)/"imatrix.dat") self.generate_importance_matrix(base_gguf, train_data_path, imatrix_path) - + # Quantize model console.print("[bold green]Quantizing model...") quantized_files: List[str] = [] @@ -1126,7 +1127,7 @@ def _convert_with_dirs(self, tmpdir: str, outdir: str) -> None: quantized_files.append(f"{quantized_name}.gguf") console.print(f"[green]Successfully quantized with {method}: {quantized_name}.gguf") - + # Upload to Hugging Face if credentials provided if self.username and self.token: repo_id = f"{self.username}/{self.model_name}-GGUF" @@ -1222,7 +1223,7 @@ def convert_command( ) -> None: """ Convert and quantize HuggingFace models to GGUF format! 🚀 - + Args: model_id (str): Your model's HF ID (like 'OEvortex/HelpingAI-Lite-1.5T') 🎯 username (str, optional): Your HF username for uploads 👤 @@ -1240,7 +1241,7 @@ def convert_command( no_lazy (bool): Disable lazy evaluation (use more RAM) 🧠 model_name (str, optional): Custom model name override 🏷️ small_first_shard (bool): Do not add tensors to the first split 📦 - + Example: >>> python -m webscout.Extra.gguf convert \\ ... -m "OEvortex/HelpingAI-Lite-1.5T" \\ diff --git a/webscout/Extra/tempmail/__init__.py b/webscout/Extra/tempmail/__init__.py index a1301a97..8c6078c4 100644 --- a/webscout/Extra/tempmail/__init__.py +++ b/webscout/Extra/tempmail/__init__.py @@ -4,19 +4,19 @@ """ from .base import ( - TempMailProvider, - AsyncTempMailProvider, - get_random_email, + AsyncTempMailProvider, + TempMailProvider, get_disposable_email, - get_provider + get_provider, + get_random_email, ) +from .emailnator import EmailnatorProvider from .mail_tm import MailTM, MailTMAsync from .temp_mail_io import TempMailIO, TempMailIOAsync -from .emailnator import EmailnatorProvider __all__ = [ 'TempMailProvider', - 'AsyncTempMailProvider', + 'AsyncTempMailProvider', 'MailTM', 'MailTMAsync', 'TempMailIO', @@ -25,4 +25,4 @@ 'get_random_email', 'get_disposable_email', 'get_provider' -] \ No newline at end of file +] diff --git a/webscout/Extra/tempmail/async_utils.py b/webscout/Extra/tempmail/async_utils.py index f0e4b438..69c230d5 100644 --- a/webscout/Extra/tempmail/async_utils.py +++ b/webscout/Extra/tempmail/async_utils.py @@ -2,61 +2,64 @@ Async utilities for temporary email generation """ import asyncio -from litprinter import ic from typing import Dict, List, Optional, Tuple -from .base import TempMailAPI, MessageResponseModel +from litprinter import ic + +from .base import TempMailAPI + class AsyncTempMailHelper: """ Async helper class for TempMail.io API Provides simplified methods for async usage of TempMail.io """ - + def __init__(self): self.api = None self.email = None self.token = None - + async def create(self, alias: Optional[str] = None, domain: Optional[str] = None) -> Tuple[str, str]: """ Create a new temporary email - + Args: alias: Optional alias for the email domain: Optional domain for the email - + Returns: Tuple containing the email address and token """ self.api = TempMailAPI() await self.api.initialize() - + try: result = await self.api.create_email(alias, domain) self.email = result.email self.token = result.token return self.email, self.token except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Error creating email: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Error creating email: {e}") await self.close() raise - + async def get_messages(self) -> List[Dict]: """ Get messages for the current email - + Returns: List of message dictionaries """ if not self.api or not self.email: raise ValueError("No email created yet") - + try: messages = await self.api.get_messages(self.email) if not messages: return [] - + return [ { "id": msg.id, @@ -70,29 +73,32 @@ async def get_messages(self) -> List[Dict]: for msg in messages ] except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Error getting messages: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Error getting messages: {e}") return [] - + async def delete(self) -> bool: """ Delete the current temporary email - + Returns: True if deletion was successful, False otherwise """ if not self.api or not self.email or not self.token: - ic.configureOutput(prefix='WARNING| '); ic("No email to delete") + ic.configureOutput(prefix='WARNING| ') + ic("No email to delete") return False - + try: result = await self.api.delete_email(self.email, self.token) return result except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Error deleting email: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Error deleting email: {e}") return False finally: await self.close() - + async def close(self) -> None: """Close the API connection""" if self.api: @@ -103,11 +109,11 @@ async def close(self) -> None: async def get_temp_email(alias: Optional[str] = None, domain: Optional[str] = None) -> Tuple[str, AsyncTempMailHelper]: """ Get a temporary email address asynchronously - + Args: alias: Optional alias for the email domain: Optional domain for the email - + Returns: Tuple containing the email address and the TempMail helper instance """ @@ -119,21 +125,21 @@ async def get_temp_email(alias: Optional[str] = None, domain: Optional[str] = No async def wait_for_message(helper: AsyncTempMailHelper, timeout: int = 60, check_interval: int = 5) -> Optional[Dict]: """ Wait for a message to arrive in the inbox - + Args: helper: The TempMail helper instance timeout: Maximum time to wait in seconds check_interval: Time between checks in seconds - + Returns: The first message if one arrives, None otherwise """ start_time = asyncio.get_event_loop().time() - + while asyncio.get_event_loop().time() - start_time < timeout: messages = await helper.get_messages() if messages: return messages[0] await asyncio.sleep(check_interval) - - return None \ No newline at end of file + + return None diff --git a/webscout/Extra/tempmail/base.py b/webscout/Extra/tempmail/base.py index 12dabc30..83b04c34 100644 --- a/webscout/Extra/tempmail/base.py +++ b/webscout/Extra/tempmail/base.py @@ -3,10 +3,10 @@ Abstract base classes for tempmail providers """ -from abc import ABC, abstractmethod -import string import random -from typing import Dict, List, Any, Optional, Union, Tuple, AsyncGenerator, Generator +import string +from abc import ABC, abstractmethod +from typing import Dict, List, Optional, Tuple, Union # Constants for email generation EMAIL_LENGTH = 16 @@ -158,4 +158,4 @@ def get_random_email(provider_name: str = "mailtm") -> Tuple[str, TempMailProvid def get_disposable_email() -> Tuple[str, TempMailProvider]: """Alias for get_random_email""" - return get_random_email() \ No newline at end of file + return get_random_email() diff --git a/webscout/Extra/tempmail/cli.py b/webscout/Extra/tempmail/cli.py index 5bf53702..075703a5 100644 --- a/webscout/Extra/tempmail/cli.py +++ b/webscout/Extra/tempmail/cli.py @@ -4,8 +4,11 @@ import sys import time + from rich.console import Console + from webscout.swiftcli import CLI, option + from .base import get_provider # Initialize console for rich output @@ -184,4 +187,4 @@ def main(): sys.exit(1) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/webscout/Extra/tempmail/emailnator.py b/webscout/Extra/tempmail/emailnator.py index 1457aa77..8950f37f 100644 --- a/webscout/Extra/tempmail/emailnator.py +++ b/webscout/Extra/tempmail/emailnator.py @@ -3,13 +3,16 @@ Synchronous provider for Emailnator.com """ -from typing import List, Dict +from json import loads from time import sleep +from typing import Dict, List + from requests import Session + from webscout.litagent import LitAgent + from .base import TempMailProvider -from json import loads -from re import findall + class EmailnatorProvider(TempMailProvider): def __init__(self): diff --git a/webscout/Extra/tempmail/mail_tm.py b/webscout/Extra/tempmail/mail_tm.py index 1321ec76..890cfe1b 100644 --- a/webscout/Extra/tempmail/mail_tm.py +++ b/webscout/Extra/tempmail/mail_tm.py @@ -3,16 +3,20 @@ Based on mail.tm and mail.gw APIs """ -import string -import requests import json import random -import time -import asyncio -from typing import List, Dict, Any, Optional, Union, Tuple +from typing import Dict, List, Optional, Tuple + +import requests from ...scout import Scout -from .base import TempMailProvider, AsyncTempMailProvider, EMAIL_LENGTH, PASSWORD_LENGTH, generate_random_string +from .base import ( + EMAIL_LENGTH, + PASSWORD_LENGTH, + AsyncTempMailProvider, + TempMailProvider, + generate_random_string, +) class MailTM(TempMailProvider): @@ -20,11 +24,11 @@ class MailTM(TempMailProvider): Mail.TM API client for temporary email services Implements the synchronous TempMailProvider interface """ - + def __init__(self, auto_create=False): """ Initialize MailTM client - + Args: auto_create (bool): Automatically create an email upon initialization """ @@ -35,17 +39,17 @@ def __init__(self, auto_create=False): self.url_domain = f"{self.url_base}/domains" self.url_msg = f"{self.url_base}/messages" self.url_token = f"{self.url_base}/token" - + self.email = None self.password = None self.token = None self.account_id = None self.header = None self.messages_count = 0 - + if auto_create: self.create_account() - + def get_domain(self) -> str: """Get available domain for email creation""" try: @@ -55,17 +59,17 @@ def get_domain(self) -> str: return ans['hydra:member'][0]['domain'] except requests.exceptions.HTTPError: return "" - + def create_account(self) -> bool: """Create a new temporary email account""" domain = self.get_domain() if not domain: return False - + # Generate random email and password self.email = generate_random_string(EMAIL_LENGTH) + '@' + domain self.password = generate_random_string(PASSWORD_LENGTH, include_digits=True) - + # Register the account myobj = {'address': self.email, "password": self.password} try: @@ -73,19 +77,19 @@ def create_account(self) -> bool: resp.raise_for_status() ans = json.loads(str(resp.text)) self.account_id = ans['id'] - + # Get token self.get_token() return True if self.token else False - + except requests.exceptions.HTTPError: return False - + def get_token(self) -> str: """Get authentication token""" if not self.email or not self.password: return "" - + myobj = {'address': self.email, "password": self.password} try: resp = requests.post(self.url_token, json=myobj) @@ -96,36 +100,36 @@ def get_token(self) -> str: return self.token except requests.exceptions.HTTPError: return "" - + def get_message_detail(self, msg_id: str) -> str: """Get detailed content of a message""" if not self.header: return "" - + try: resp = requests.get(f"{self.url_msg}/{msg_id}", headers=self.header) resp.raise_for_status() ans = json.loads(str(resp.text)) - + # Use Scout instead of BeautifulSoup for HTML parsing scout = Scout(ans['text']) - + # Extract text with Scout's get_text method return scout.get_text(strip=True) - + except requests.exceptions.HTTPError: return "" - + def get_messages(self) -> List[Dict]: """Get messages from the inbox""" if not self.header: return [] - + try: resp = requests.get(f"{self.url_msg}?page=1", headers=self.header) resp.raise_for_status() ans = json.loads(str(resp.text)) - + messages = [] if ans['hydra:totalItems'] > 0: for x in ans['hydra:member']: @@ -142,27 +146,27 @@ def get_messages(self) -> List[Dict]: return messages except requests.exceptions.HTTPError: return [] - + def check_new_messages(self) -> List[Dict]: """Check for new messages and return only the new ones""" messages = self.get_messages() if not messages: return [] - + if len(messages) > self.messages_count: new_msg_count = len(messages) - self.messages_count new_messages = messages[:new_msg_count] self.messages_count = len(messages) return new_messages - + self.messages_count = len(messages) return [] - + def delete_account(self) -> bool: """Delete the current temporary email account""" if not self.header or not self.account_id: return False - + try: resp = requests.delete(f"{self.url_accounts}/{self.account_id}", headers=self.header) resp.raise_for_status() @@ -175,12 +179,12 @@ def delete_account(self) -> bool: return True except requests.exceptions.HTTPError: return False - + def get_account_info(self) -> Dict: """Get current account information""" if not self.header: return {} - + try: resp = requests.get(self.url_me, headers=self.header) resp.raise_for_status() @@ -194,7 +198,7 @@ class MailTMAsync(AsyncTempMailProvider): Asynchronous Mail.TM API client for temporary email services Implements the AsyncTempMailProvider interface """ - + def __init__(self): """Initialize MailTM Async client""" self.url_bases = ['https://api.mail.tm', 'https://api.mail.gw'] @@ -204,39 +208,39 @@ def __init__(self): self.url_domain = f"{self.url_base}/domains" self.url_msg = f"{self.url_base}/messages" self.url_token = f"{self.url_base}/token" - + self.session = None self.email = None self.password = None self.token = None self.account_id = None self.header = None - + async def initialize(self): """Initialize the session""" import aiohttp self.session = aiohttp.ClientSession() return self - + async def close(self) -> None: """Close the session""" if self.session: await self.session.close() self.session = None - + async def __aenter__(self): """Context manager entry""" return await self.initialize() - + async def __aexit__(self, exc_type, exc_val, exc_tb): """Context manager exit""" await self.close() - + async def get_domain(self) -> str: """Get available domain for email creation""" if not self.session: await self.initialize() - + try: async with self.session.get(f"{self.url_domain}?page=1") as resp: resp.raise_for_status() @@ -244,26 +248,26 @@ async def get_domain(self) -> str: return ans['hydra:member'][0]['domain'] except Exception: return "" - + async def create_email(self, alias: Optional[str] = None, domain: Optional[str] = None) -> Tuple[str, str]: """Create a new email account""" if not self.session: await self.initialize() - + if not domain: domain = await self.get_domain() if not domain: return "", "" - + # Generate random email address or use alias if alias: self.email = f"{alias}@{domain}" else: self.email = f"{generate_random_string(EMAIL_LENGTH)}@{domain}" - + # Generate password self.password = generate_random_string(PASSWORD_LENGTH, include_digits=True) - + # Register account data = {'address': self.email, 'password': self.password} try: @@ -271,19 +275,19 @@ async def create_email(self, alias: Optional[str] = None, domain: Optional[str] resp.raise_for_status() ans = await resp.json() self.account_id = ans['id'] - + # Get token token = await self._get_token() return self.email, token - + except Exception: return "", "" - + async def _get_token(self) -> str: """Get authentication token""" if not self.email or not self.password: return "" - + data = {'address': self.email, 'password': self.password} try: async with self.session.post(self.url_token, json=data) as resp: @@ -294,37 +298,37 @@ async def _get_token(self) -> str: return self.token except Exception: return "" - + async def get_message_detail(self, msg_id: str) -> str: """Get detailed content of a message""" if not self.header: return "" - + try: async with self.session.get(f"{self.url_msg}/{msg_id}", headers=self.header) as resp: resp.raise_for_status() ans = await resp.json() - + # Use Scout instead of BeautifulSoup for HTML parsing scout = Scout(ans['text']) - + # Extract text with Scout's get_text method with improved options # Strip whitespace for cleaner output return scout.get_text(separator=' ', strip=True) - + except Exception: return "" - + async def get_messages(self) -> List[Dict]: """Get messages for a temporary email""" if not self.header: return [] - + try: async with self.session.get(f"{self.url_msg}?page=1", headers=self.header) as resp: resp.raise_for_status() ans = await resp.json() - + messages = [] if ans['hydra:totalItems'] > 0: for x in ans['hydra:member']: @@ -342,12 +346,12 @@ async def get_messages(self) -> List[Dict]: return messages except Exception: return [] - + async def delete_email(self) -> bool: """Delete a temporary email""" if not self.header or not self.account_id: return False - + try: async with self.session.delete(f"{self.url_accounts}/{self.account_id}", headers=self.header) as resp: resp.raise_for_status() @@ -358,4 +362,4 @@ async def delete_email(self) -> bool: self.header = None return True except Exception: - return False \ No newline at end of file + return False diff --git a/webscout/Extra/tempmail/temp_mail_io.py b/webscout/Extra/tempmail/temp_mail_io.py index f4299f9d..bbc52832 100644 --- a/webscout/Extra/tempmail/temp_mail_io.py +++ b/webscout/Extra/tempmail/temp_mail_io.py @@ -3,12 +3,12 @@ Based on temp-mail.io API """ -import aiohttp -import asyncio from dataclasses import dataclass -from typing import List, Dict, Any, Optional, Union, Tuple, NoReturn +from typing import Any, Dict, List, Optional, Tuple + +import aiohttp -from .base import AsyncTempMailProvider, TempMailProvider, generate_random_string +from .base import AsyncTempMailProvider, TempMailProvider @dataclass @@ -289,4 +289,4 @@ def get_account_info(self) -> Dict: return { 'email': self.email, 'token': self.token - } \ No newline at end of file + } diff --git a/webscout/Extra/weather.py b/webscout/Extra/weather.py index 435bd03a..b2b00735 100644 --- a/webscout/Extra/weather.py +++ b/webscout/Extra/weather.py @@ -5,17 +5,18 @@ from the wttr.in service with proper typing and a consistent interface. """ -import requests from datetime import datetime -from typing import List, Dict, Any, Optional +from typing import Any, Dict, List, Optional + +import requests class CurrentCondition: """Current weather conditions with strongly typed properties.""" - + def __init__(self, data: Dict[str, Any]) -> None: """Initialize with current condition data. - + Args: data: Current condition data dictionary from wttr.in """ @@ -35,10 +36,10 @@ def __init__(self, data: Dict[str, Any]) -> None: class Location: """Location information with strongly typed properties.""" - + def __init__(self, data: Dict[str, Any]) -> None: """Initialize with location data. - + Args: data: Location data dictionary from wttr.in """ @@ -51,10 +52,10 @@ def __init__(self, data: Dict[str, Any]) -> None: class HourlyForecast: """Hourly forecast information with strongly typed properties.""" - + def __init__(self, data: Dict[str, Any]) -> None: """Initialize with hourly forecast data. - + Args: data: Hourly forecast data dictionary from wttr.in """ @@ -73,10 +74,10 @@ def __init__(self, data: Dict[str, Any]) -> None: class DayForecast: """Daily forecast information with strongly typed properties.""" - + def __init__(self, data: Dict[str, Any]) -> None: """Initialize with daily forecast data. - + Args: data: Daily forecast data dictionary from wttr.in """ @@ -87,7 +88,7 @@ def __init__(self, data: Dict[str, Any]) -> None: self.date_formatted = datetime.strptime(self.date, '%Y-%m-%d').strftime('%a, %b %d') except ValueError: pass - + self.max_temp_c: Optional[str] = data.get('maxtempC') self.max_temp_f: Optional[str] = data.get('maxtempF') self.min_temp_c: Optional[str] = data.get('mintempC') @@ -95,7 +96,7 @@ def __init__(self, data: Dict[str, Any]) -> None: self.avg_temp_c: Optional[str] = data.get('avgtempC') self.avg_temp_f: Optional[str] = data.get('avgtempF') self.sun_hour: Optional[str] = data.get('sunHour') - + # Parse astronomy data (simplified) if data.get('astronomy') and len(data.get('astronomy', [])) > 0: astro = data.get('astronomy', [{}])[0] @@ -104,7 +105,7 @@ def __init__(self, data: Dict[str, Any]) -> None: self.moon_phase: Optional[str] = astro.get('moon_phase') else: self.sunrise = self.sunset = self.moon_phase = None - + # Parse hourly forecasts self.hourly: List[HourlyForecast] = [] for hour_data in data.get('hourly', []): @@ -113,10 +114,10 @@ def __init__(self, data: Dict[str, Any]) -> None: class Weather: """Weather response object with strongly typed properties.""" - + def __init__(self, data: Optional[Dict[str, Any]] = None) -> None: """Initialize with weather data. - + Args: data: Weather data dictionary from wttr.in """ @@ -125,50 +126,50 @@ def __init__(self, data: Optional[Dict[str, Any]] = None) -> None: self.location = None self.forecast_days = [] return - + # Parse current condition self.current_condition: Optional[CurrentCondition] = None if data.get('current_condition') and len(data.get('current_condition', [])) > 0: self.current_condition = CurrentCondition(data.get('current_condition', [{}])[0]) - + # Parse location self.location: Optional[Location] = None if data.get('nearest_area') and len(data.get('nearest_area', [])) > 0: self.location = Location(data.get('nearest_area', [{}])[0]) - + # Parse forecast days self.forecast_days: List[DayForecast] = [] for day_data in data.get('weather', []): self.forecast_days.append(DayForecast(day_data)) - + @property def today(self) -> Optional[DayForecast]: """Get today's forecast.""" return self.forecast_days[0] if self.forecast_days else None - + @property def tomorrow(self) -> Optional[DayForecast]: """Get tomorrow's forecast.""" return self.forecast_days[1] if len(self.forecast_days) > 1 else None - + @property def summary(self) -> str: """Get a simple text summary of current weather.""" if not self.current_condition or not self.location: return "Weather data not available" - + return f"{self.location.name}, {self.location.country}: {self.current_condition.weather_desc}, {self.current_condition.temp_c}°C ({self.current_condition.temp_f}°F)" class WeatherClient: """Client for fetching weather information.""" - + def get_weather(self, location: str) -> Weather: """Get weather for the specified location. - + Args: location: Location to get weather for (city name, zip code, etc.) - + Returns: Weather object containing all weather data """ @@ -183,12 +184,12 @@ def get_weather(self, location: str) -> Weather: def get(location: str) -> Weather: """Convenience function to get weather for a location. - + Args: location: Location to get weather for - + Returns: Weather object containing all weather data """ client = WeatherClient() - return client.get_weather(location) \ No newline at end of file + return client.get_weather(location) diff --git a/webscout/Extra/weather_ascii.py b/webscout/Extra/weather_ascii.py index 9f8fc6fa..7920a286 100644 --- a/webscout/Extra/weather_ascii.py +++ b/webscout/Extra/weather_ascii.py @@ -5,26 +5,27 @@ in ASCII art format using the wttr.in service. """ +from typing import Any, Dict, Optional + import requests -from typing import Dict, Optional, Any class WeatherAscii: """Container for ASCII weather data with a simple API.""" - + def __init__(self, content: str) -> None: """Initialize with ASCII weather content. - + Args: content: ASCII weather data or error message """ self._content = content - + @property def content(self) -> str: """Get the ASCII content, similar to choices.message.content in OpenAI API.""" return self._content - + def __str__(self) -> str: """String representation of ASCII weather.""" return self.content @@ -32,24 +33,24 @@ def __str__(self) -> str: class WeatherAsciiClient: """Client for fetching weather information in ASCII art.""" - + def get_weather(self, location: str, params: Optional[Dict[str, Any]] = None) -> WeatherAscii: """Get ASCII weather for a location. - + Args: location: The location for which to fetch weather data params: Additional parameters for the request - + Returns: WeatherAscii object containing ASCII art weather data """ url = f"https://wttr.in/{location}" headers = {'User-Agent': 'curl'} - + try: response = requests.get(url, headers=headers, params=params, timeout=10) response.raise_for_status() - + if response.status_code == 200: # Remove the footer line from wttr.in ascii_weather = "\n".join(response.text.splitlines()[:-1]) @@ -63,11 +64,11 @@ def get_weather(self, location: str, params: Optional[Dict[str, Any]] = None) -> def get(location: str, params: Optional[Dict[str, Any]] = None) -> WeatherAscii: """Convenience function to get ASCII weather for a location. - + Args: location: Location to get weather for params: Additional parameters for the request - + Returns: WeatherAscii object containing ASCII art weather data """ diff --git a/webscout/Provider/AISEARCH/PERPLEXED_search.py b/webscout/Provider/AISEARCH/PERPLEXED_search.py index d6236f0f..806a858c 100644 --- a/webscout/Provider/AISEARCH/PERPLEXED_search.py +++ b/webscout/Provider/AISEARCH/PERPLEXED_search.py @@ -1,9 +1,9 @@ +from typing import Dict, Generator, Optional, Union + import requests -import json -from typing import Any, Dict, Generator, Optional, Union -from webscout.AIbase import AISearch, SearchResponse from webscout import exceptions +from webscout.AIbase import AISearch, SearchResponse from webscout.litagent import LitAgent from webscout.sanitize import sanitize_stream @@ -181,5 +181,8 @@ def for_non_stream(): if __name__ == "__main__": ai = PERPLEXED() response = ai.search("What is Python?", stream=True, raw=False) - for chunks in response: - print(chunks, end="", flush=True) + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes, SearchResponse)): + for chunks in response: + print(chunks, end="", flush=True) + else: + print(response) diff --git a/webscout/Provider/AISEARCH/Perplexity.py b/webscout/Provider/AISEARCH/Perplexity.py index a41d61e4..fedb34fd 100644 --- a/webscout/Provider/AISEARCH/Perplexity.py +++ b/webscout/Provider/AISEARCH/Perplexity.py @@ -1,12 +1,13 @@ import json import random import time +from typing import Any, Dict, Generator, List, Optional, Union from uuid import uuid4 -from typing import Dict, Optional, Generator, Union, Any, List + from curl_cffi import requests -from webscout.AIbase import AISearch, SearchResponse from webscout import exceptions +from webscout.AIbase import AISearch, SearchResponse from webscout.litagent import LitAgent from webscout.sanitize import sanitize_stream @@ -79,7 +80,7 @@ def __init__( data='40{"jwt":"anonymous-ask-user"}', ) self.session.get("https://www.perplexity.ai/api/auth/session") - except: + except Exception: pass def _extract_answer(self, response): @@ -97,7 +98,7 @@ def _extract_answer(self, response): if isinstance(answer_content, str): try: return json.loads(answer_content).get("answer", "") - except: + except Exception: return str(answer_content) elif isinstance(answer_content, dict): return answer_content.get("answer", "") @@ -116,14 +117,14 @@ def _extract_answer(self, response): if isinstance(answer_content, str): try: return json.loads(answer_content).get("answer", "") - except: + except Exception: return str(answer_content) elif isinstance(answer_content, dict): return answer_content.get("answer", "") elif isinstance(text_val, str): try: return json.loads(text_val).get("answer", text_val) - except: + except Exception: return text_val return "" @@ -141,14 +142,14 @@ def _extract_answer(self, response): if isinstance(answer_content, str): try: return json.loads(answer_content).get("answer", "") - except: + except Exception: return str(answer_content) elif isinstance(answer_content, dict): return answer_content.get("answer", "") elif isinstance(text_val, str): try: return json.loads(text_val).get("answer", text_val) - except: + except Exception: return text_val return "" @@ -164,12 +165,12 @@ def _extract_answer(self, response): if isinstance(content, dict) and "answer" in content: try: return json.loads(content["answer"]).get("answer", "") - except: + except Exception: return str(content["answer"]) elif isinstance(text_val, str): try: return json.loads(text_val).get("answer", text_val) - except: + except Exception: return text_val return "" @@ -193,7 +194,7 @@ def search( sources = ["web"] # Prepare request data - json_data = { + json_data: Dict[str, Any] = { "query_str": prompt, "params": { "attachments": follow_up["attachments"] if follow_up else [], @@ -267,7 +268,7 @@ def extract_perplexity_content(data): else: if raw: return resp.text - + full_response_text = "" for chunk in stream_response(): full_response_text += str(chunk) diff --git a/webscout/Provider/AISEARCH/__init__.py b/webscout/Provider/AISEARCH/__init__.py index e1b423a4..59c4dd89 100644 --- a/webscout/Provider/AISEARCH/__init__.py +++ b/webscout/Provider/AISEARCH/__init__.py @@ -1,8 +1,8 @@ -from webscout.Provider.AISEARCH.PERPLEXED_search import PERPLEXED -from webscout.Provider.AISEARCH.Perplexity import Perplexity from webscout.Provider.AISEARCH.genspark_search import Genspark from webscout.Provider.AISEARCH.iask_search import IAsk from webscout.Provider.AISEARCH.monica_search import Monica +from webscout.Provider.AISEARCH.PERPLEXED_search import PERPLEXED +from webscout.Provider.AISEARCH.Perplexity import Perplexity from webscout.Provider.AISEARCH.webpilotai_search import webpilotai # List of all exported names diff --git a/webscout/Provider/AISEARCH/genspark_search.py b/webscout/Provider/AISEARCH/genspark_search.py index c0663fe3..166c2ba3 100644 --- a/webscout/Provider/AISEARCH/genspark_search.py +++ b/webscout/Provider/AISEARCH/genspark_search.py @@ -1,13 +1,12 @@ -import cloudscraper +import sys +from typing import Any, Dict, Iterator, List, Optional, TypedDict, Union, cast from uuid import uuid4 -import json -from typing import TypedDict, List, Iterator, cast, Dict, Optional, Union, Any + +import cloudscraper import requests -import sys -import re -from webscout.AIbase import AISearch, SearchResponse from webscout import exceptions +from webscout.AIbase import AISearch, SearchResponse from webscout.litagent import LitAgent from webscout.sanitize import sanitize_stream @@ -18,9 +17,10 @@ class SourceDict(TypedDict, total=False): snippet: str favicon: str -class StatusUpdateDict(TypedDict): +class StatusUpdateDict(TypedDict, total=False): type: str message: str + data: Any class StatusTopBarDict(TypedDict, total=False): type: str @@ -41,7 +41,7 @@ class ResultSummaryDict(TypedDict, total=False): class Genspark(AISearch): """ Strongly typed Genspark AI search API client. - + Updated to handle Unicode more gracefully and support more result fields. """ @@ -130,7 +130,7 @@ def search( ]: """ Strongly typed search method for Genspark API. - + Args: prompt: The search query or prompt. stream: If True, yields results as they arrive. @@ -138,7 +138,7 @@ def search( """ self._reset_search_data() url = f"{self.chat_endpoint}?query={requests.utils.quote(prompt)}" - + def _process_stream() -> Iterator[Union[dict, SearchResponse]]: #type: ignore CloudflareException = cloudscraper.exceptions.CloudflareException RequestException = requests.exceptions.RequestException @@ -159,9 +159,9 @@ def _extract_genspark_content(data: dict) -> Optional[str]: event_type = data.get("type") field_name = data.get("field_name") result_id = data.get("result_id") - + # Internal State Updates - if event_type == "result_start": + if event_type == "result_start" and isinstance(result_id, str): self.result_summary[result_id] = cast(ResultSummaryDict, { "source": data.get("result_source"), "rel_score": data.get("result_rel_score"), @@ -207,7 +207,7 @@ def _extract_genspark_content(data: dict) -> Optional[str]: # Content extraction if event_type == "result_field_delta" and field_name: if ( - field_name.startswith("streaming_detail_answer") or + field_name.startswith("streaming_detail_answer") or field_name.startswith("streaming_simple_answer") or field_name == "answer" or field_name == "content" @@ -225,7 +225,7 @@ def _extract_genspark_content(data: dict) -> Optional[str]: raw=raw, output_formatter=None if raw else lambda x: SearchResponse(x) if isinstance(x, str) else x, ) - + for item in processed_stream: yield item except CloudflareException as e: @@ -234,7 +234,7 @@ def _extract_genspark_content(data: dict) -> Optional[str]: raise exceptions.APIConnectionError(f"Request failed: {e}") processed_stream_gen = _process_stream() - + if stream: return processed_stream_gen else: @@ -246,7 +246,7 @@ def _extract_genspark_content(data: dict) -> Optional[str]: else: if isinstance(item, SearchResponse): full_SearchResponse_text += str(item) - + if raw: full_raw = "".join(str(item) for item in all_raw_events_for_this_search) self.last_response = full_raw @@ -261,18 +261,21 @@ def _extract_genspark_content(data: dict) -> Optional[str]: ai = Genspark() try: search_result_stream = ai.search("liger-kernal details", stream=True, raw=False) - - for chunk in search_result_stream: - try: - # Use a more robust way to print Unicode characters on Windows - text = str(chunk) - sys.stdout.write(text) - sys.stdout.flush() - except UnicodeEncodeError: - # Fallback for Windows consoles that don't support UTF-8 - safe_chunk = str(chunk).encode('ascii', errors='replace').decode('ascii') - sys.stdout.write(safe_chunk) - sys.stdout.flush() + + if hasattr(search_result_stream, "__iter__") and not isinstance(search_result_stream, (str, bytes, SearchResponse)): + for chunk in search_result_stream: + try: + # Use a more robust way to print Unicode characters on Windows + text = str(chunk) + sys.stdout.write(text) + sys.stdout.flush() + except UnicodeEncodeError: + # Fallback for Windows consoles that don't support UTF-8 + safe_chunk = str(chunk).encode('ascii', errors='replace').decode('ascii') + sys.stdout.write(safe_chunk) + sys.stdout.flush() + else: + print(search_result_stream) print() except KeyboardInterrupt: diff --git a/webscout/Provider/AISEARCH/iask_search.py b/webscout/Provider/AISEARCH/iask_search.py index 1c6b4639..72474782 100644 --- a/webscout/Provider/AISEARCH/iask_search.py +++ b/webscout/Provider/AISEARCH/iask_search.py @@ -1,13 +1,14 @@ -import aiohttp import asyncio -import lxml.html import re import urllib.parse +from typing import AsyncIterator, Dict, Generator, Literal, Optional, Union + +import aiohttp +import lxml.html from markdownify import markdownify as md -from typing import Dict, Optional, Generator, Union, AsyncIterator, Literal -from webscout.AIbase import AISearch, SearchResponse from webscout import exceptions +from webscout.AIbase import AISearch, SearchResponse from webscout.scout import Scout @@ -240,26 +241,31 @@ def sync_generator(): async_gen = loop.run_until_complete(async_gen_coro) # Process chunks one by one - while True: - try: - # Get the next chunk - chunk_coro = async_gen.__anext__() - chunk = loop.run_until_complete(chunk_coro) - - # Update buffer and yield the chunk - if isinstance(chunk, dict) and 'text' in chunk: - buffer += chunk['text'] - elif isinstance(chunk, SearchResponse): - buffer += chunk.text - else: - buffer += str(chunk) - - yield chunk - except StopAsyncIteration: - break - except Exception as e: - print(f"Error in generator: {e}") - break + if hasattr(async_gen, "__anext__"): + while True: + try: + # Get the next chunk + chunk_coro = async_gen.__anext__() + chunk = loop.run_until_complete(chunk_coro) + + # Update buffer and yield the chunk + if isinstance(chunk, dict) and 'text' in chunk: + buffer += chunk['text'] + elif isinstance(chunk, SearchResponse): + buffer += chunk.text + else: + buffer += str(chunk) + + yield chunk + except StopAsyncIteration: + break + except Exception as e: + print(f"Error in generator: {e}") + break + elif isinstance(async_gen, SearchResponse): + yield async_gen + else: + yield str(async_gen) finally: # Store the final response and close the loop self.last_response = {"text": buffer} @@ -337,7 +343,7 @@ async def stream_generator() -> AsyncIterator[str]: yield formatted_chunk else: yield chunk.replace("
", "\n") - except: + except Exception: cache = cache_find(diff) if cache: if diff.get("response", None): diff --git a/webscout/Provider/AISEARCH/monica_search.py b/webscout/Provider/AISEARCH/monica_search.py index 7f0443ef..909a78ef 100644 --- a/webscout/Provider/AISEARCH/monica_search.py +++ b/webscout/Provider/AISEARCH/monica_search.py @@ -1,9 +1,9 @@ +from typing import Dict, Generator, Optional, Union + import requests -import json -from typing import Any, Dict, Generator, Optional, Union -from webscout.AIbase import AISearch, SearchResponse from webscout import exceptions +from webscout.AIbase import AISearch, SearchResponse from webscout.litagent import LitAgent from webscout.sanitize import sanitize_stream @@ -208,5 +208,8 @@ def for_non_stream(): if __name__ == "__main__": ai = Monica() response = ai.search("What is Python?", stream=True, raw=False) - for chunks in response: - print(chunks, end="", flush=True) + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes, SearchResponse)): + for chunks in response: + print(chunks, end="", flush=True) + else: + print(response) diff --git a/webscout/Provider/AISEARCH/webpilotai_search.py b/webscout/Provider/AISEARCH/webpilotai_search.py index 58ef9f48..d503c074 100644 --- a/webscout/Provider/AISEARCH/webpilotai_search.py +++ b/webscout/Provider/AISEARCH/webpilotai_search.py @@ -1,21 +1,21 @@ -import requests -import json import re -from typing import Dict, Optional, Generator, Union, Any, List +from typing import Any, Dict, Generator, List, Optional, Union + +import requests -from webscout.AIbase import AISearch, SearchResponse from webscout import exceptions +from webscout.AIbase import AISearch, SearchResponse from webscout.litagent import LitAgent from webscout.sanitize import sanitize_stream class webpilotai(AISearch): """A class to interact with the webpilotai (WebPilot) AI search API. - - webpilotai provides a web-based comprehensive search SearchResponse interface that returns AI-generated + + webpilotai provides a web-based comprehensive search SearchResponse interface that returns AI-generated SearchResponses with source references and related questions. It supports both streaming and non-streaming SearchResponses. - + Basic Usage: >>> from webscout import webpilotai >>> ai = webpilotai() @@ -23,18 +23,18 @@ class webpilotai(AISearch): >>> response = ai.search("What is Python?") >>> print(response) Python is a high-level programming language... - + >>> # Streaming example >>> for chunk in ai.search("Tell me about AI", stream=True): ... print(chunk, end="", flush=True) Artificial Intelligence is... - + >>> # Raw SearchResponse format >>> for chunk in ai.search("Hello", stream=True, raw=True): ... print(chunk) {'text': 'Hello'} {'text': ' there!'} - + Args: timeout (int, optional): Request timeout in seconds. Defaults to 90. proxies (dict, optional): Proxy configuration for requests. Defaults to None. @@ -46,7 +46,7 @@ def __init__( proxies: Optional[dict] = None, ): """Initialize the webpilotai API client. - + Args: timeout (int, optional): Request timeout in seconds. Defaults to 90. proxies (dict, optional): Proxy configuration for requests. Defaults to None. @@ -55,7 +55,7 @@ def __init__( self.api_endpoint = "https://api.webpilotai.com/rupee/v1/search" self.timeout = timeout self.last_response = {} - + # The 'Bearer null' is part of the API's expected headers self.headers = { 'Accept': 'application/json, text/plain, */*, text/event-stream', @@ -65,7 +65,7 @@ def __init__( 'Referer': 'https://www.webpilot.ai/', 'User-Agent': LitAgent().random(), } - + self.session.headers.update(self.headers) self.proxies = proxies @@ -76,10 +76,10 @@ def search( raw: bool = False, ) -> Union[SearchResponse, Generator[Union[Dict[str, str], SearchResponse], None, None], List[Any]]: """Search using the webpilotai API and get AI-generated SearchResponses. - + This method sends a search query to webpilotai and returns the AI-generated SearchResponse. It supports both streaming and non-streaming modes, as well as raw SearchResponse format. - + Args: prompt (str): The search query or prompt to send to the API. stream (bool, optional): If True, yields SearchResponse chunks as they arrive. @@ -87,12 +87,12 @@ def search( raw (bool, optional): If True, returns raw SearchResponse dictionaries with 'text' key. If False, returns SearchResponse objects that convert to text automatically. Defaults to False. - + Returns: - Union[SearchResponse, Generator[Union[Dict[str, str], SearchResponse], None, None]]: + Union[SearchResponse, Generator[Union[Dict[str, str], SearchResponse], None, None]]: - If stream=False: Returns complete SearchResponse as SearchResponse object - If stream=True: Yields SearchResponse chunks as either Dict or SearchResponse objects - + Raises: APIConnectionError: If the API request fails """ @@ -100,10 +100,9 @@ def search( "q": prompt, "threadId": "" # Empty for new search } - + def for_stream(): - full_SearchResponse_content = "" - + try: with self.session.post( self.api_endpoint, @@ -116,7 +115,7 @@ def for_stream(): raise exceptions.APIConnectionError( f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}" ) - + def extract_webpilot_content(data): if isinstance(data, dict): if data.get('type') == 'data': @@ -138,7 +137,7 @@ def extract_webpilot_content(data): for chunk in processed_chunks: yield chunk - + except requests.exceptions.Timeout: raise exceptions.APIConnectionError("Request timed out") except requests.exceptions.RequestException as e: @@ -151,7 +150,7 @@ def for_non_stream(): full_content += str(chunk) else: full_content += str(chunk) - + if raw: return full_content else: @@ -163,37 +162,37 @@ def for_non_stream(): return for_stream() else: return for_non_stream() - + @staticmethod def format_SearchResponse(text: str) -> str: """Format the SearchResponse text for better readability. - + Args: text (str): The raw SearchResponse text - + Returns: str: Formatted text with improved structure """ # Clean up formatting # Remove excessive newlines clean_text = re.sub(r'\n{3,}', '\n\n', text) - + # Ensure consistent spacing around sections clean_text = re.sub(r'([.!?])\s*\n\s*([A-Z])', r'\1\n\n\2', clean_text) - + # Clean up any leftover HTML or markdown artifacts clean_text = re.sub(r'<[^>]*>', '', clean_text) - + # Remove trailing whitespace on each line clean_text = '\n'.join(line.rstrip() for line in clean_text.split('\n')) - + return clean_text.strip() if __name__ == "__main__": from rich import print - + ai = webpilotai() r = ai.search("What is Python?", stream=True, raw=False) for chunk in r: - print(chunk, end="", flush=True) \ No newline at end of file + print(chunk, end="", flush=True) diff --git a/webscout/Provider/Algion.py b/webscout/Provider/Algion.py index 1c981545..5e921b18 100644 --- a/webscout/Provider/Algion.py +++ b/webscout/Provider/Algion.py @@ -1,23 +1,24 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError import json -from typing import Any, Dict, Optional, Generator, Union -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session + from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent + class Algion(Provider): """ A class to interact with the Algion API (OpenAI-compatible free API). - + Attributes: AVAILABLE_MODELS: List of available models url: API endpoint URL api: API key for authentication - + Examples: >>> from webscout.Provider.Algion import Algion >>> ai = Algion() @@ -25,17 +26,17 @@ class Algion(Provider): >>> print(response) """ @classmethod - def get_models(cls, api_key: str = None): + def get_models(cls, api_key: Optional[str] = None): """Fetch available models from Algion API. - + Args: api_key (str, optional): Algion API key. If not provided, uses default free key. - + Returns: list: List of available model IDs """ api_key = api_key or "123123" - + try: # Use a temporary curl_cffi session for this class method temp_session = Session() @@ -43,21 +44,21 @@ def get_models(cls, api_key: str = None): "Content-Type": "application/json", "Authorization": f"Bearer {api_key}", } - + response = temp_session.get( "https://api.algion.dev/v1/models", headers=headers, impersonate="chrome110" ) - + if response.status_code != 200: raise Exception(f"Failed to fetch models: HTTP {response.status_code}") - + data = response.json() if "data" in data and isinstance(data["data"], list): return [model["id"] for model in data["data"]] raise Exception("Invalid response format from API") - + except (CurlError, Exception) as e: raise Exception(f"Failed to fetch models: {str(e)}") @@ -78,18 +79,18 @@ def __init__( is_conversation: bool = True, max_tokens: int = 2049, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "gpt-4o", system_prompt: str = "You are a helpful assistant.", browser: str = "chrome" ): """Initializes the Algion API client. - + Args: api_key: API key for authentication (default: "123123" - free key) is_conversation: Whether to use conversation mode @@ -168,7 +169,7 @@ def refresh_identity(self, browser: str = None): Args: browser: Specific browser to use for the new fingerprint - + Returns: dict: New fingerprint """ @@ -194,9 +195,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: """ Sends a prompt to the Algion API and returns the response. @@ -209,7 +211,7 @@ def ask( Returns: Dict or Generator: Response from the API - + Examples: >>> ai = Algion() >>> response = ai.ask("What is AI?") @@ -261,7 +263,7 @@ def for_stream(): # Always yield as string, even in raw mode if isinstance(content_chunk, bytes): content_chunk = content_chunk.decode('utf-8', errors='ignore') - + if raw: yield content_chunk else: @@ -325,7 +327,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, ) -> Union[str, Generator[str, None, None]]: @@ -341,7 +343,7 @@ def chat( Returns: str or Generator: Response text - + Examples: >>> ai = Algion() >>> response = ai.chat("Tell me a joke") @@ -370,7 +372,7 @@ def for_non_stream_chat(): return for_stream_chat() if stream else for_non_stream_chat() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """ Extracts the message from the API response. @@ -380,8 +382,9 @@ def get_message(self, response: dict) -> str: Returns: str: The message content """ - assert isinstance(response, dict), "Response should be of dict data-type only" - return response["text"] + if not isinstance(response, dict): + return str(response) + return response.get("text", "") try: fetched_models = Algion.get_models() @@ -392,12 +395,12 @@ def get_message(self, response: dict) -> str: if __name__ == "__main__": from rich import print - + print("-" * 80) print(f"{'Model':<50} {'Status':<10} {'Response'}") print("-" * 80) - for model in Algion.AVAILABLE_MODELS: + for model in Algion.AVAILABLE_MODELS: try: test_ai = Algion(model=model, timeout=60) response = test_ai.chat("Say 'Hello' in one word", stream=True) diff --git a/webscout/Provider/Andi.py b/webscout/Provider/Andi.py index 9ea9c25c..5c80a5eb 100644 --- a/webscout/Provider/Andi.py +++ b/webscout/Provider/Andi.py @@ -1,14 +1,15 @@ +import json +from typing import Generator, Union from uuid import uuid4 + import requests -import json -from typing import Union, Dict, Any, Generator -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider + from webscout import exceptions -from webscout.search import DuckDuckGoSearch +from webscout.AIbase import Provider +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent +from webscout.search import DuckDuckGoSearch + class AndiSearch(Provider): required_auth = False @@ -130,7 +131,7 @@ def ask( "title": result.title, "link": result.href, "desc": result.body, - "image": "", + "image": "", "type": "website", "source": result.href.split("//")[1].split("/")[0] if "//" in result.href else result.href.split("/")[0] # Extract the domain name } @@ -231,10 +232,10 @@ def get_message(self, response: dict) -> str: """ assert isinstance(response, dict), "Response should be of dict data-type only" return response["text"] - + if __name__ == '__main__': from rich import print ai = AndiSearch() response = ai.chat("tell me about india") for chunk in response: - print(chunk, end="", flush=True) \ No newline at end of file + print(chunk, end="", flush=True) diff --git a/webscout/Provider/Apriel.py b/webscout/Provider/Apriel.py index a1994bcb..f14348cb 100644 --- a/webscout/Provider/Apriel.py +++ b/webscout/Provider/Apriel.py @@ -3,17 +3,15 @@ This provider integrates the Apriel chat model into the Webscout framework. """ -from typing import Generator, Optional, Union, Any, Dict -import json import time +from typing import Any, Dict, Generator, Optional, Union + from curl_cffi import CurlError from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers from webscout.litagent import LitAgent from webscout.sanitize import sanitize_stream @@ -40,12 +38,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 600, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, system_prompt: str = "You are a helpful assistant.", model: str = "UNKNOWN" ): @@ -162,9 +160,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: """ Sends a prompt to the Apriel Gradio API and returns the response. @@ -190,7 +189,7 @@ def ask( ) session_hash = self._get_session_hash() - event_id = self._join_queue(session_hash, conversation_prompt) + self._join_queue(session_hash, conversation_prompt) self._run_predict(session_hash) def for_stream(): @@ -247,7 +246,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, ) -> Union[str, Generator[str, None, None]]: @@ -289,22 +288,26 @@ def for_non_stream(): return for_stream() if stream else for_non_stream() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """ Extracts the message from the API response. Args: - response (dict): The API response. + response (Response): The API response. Returns: str: The message content. """ - assert isinstance(response, dict), "Response should be of dict data-type only" + if not isinstance(response, dict): + return str(response) return response.get("text", "") if __name__ == "__main__": from rich import print ai = Apriel(timeout=60) response = ai.chat("write a poem about AI", stream=True, raw=False) - for chunk in response: - print(chunk, end="", flush=True) + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end="", flush=True) + else: + print(response) diff --git a/webscout/Provider/Ayle.py b/webscout/Provider/Ayle.py index ff5bdb5c..b3b6ac9b 100644 --- a/webscout/Provider/Ayle.py +++ b/webscout/Provider/Ayle.py @@ -1,21 +1,28 @@ -from curl_cffi import CurlError -from curl_cffi.requests import Session, Response # Import Response import json import uuid -from typing import Any, Dict, Union, Optional, List, Generator -from datetime import datetime -from webscout.AIutel import Optimizers, Conversation, AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Response as CurlResponse # Import Response +from curl_cffi.requests import Session + from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) from webscout.litagent import LitAgent # Model configurations -MODEL_CONFIGS = { +MODEL_CONFIGS: Dict[str, Dict[str, Any]] = { "ayle": { "endpoint": "https://ayle.chat/api/chat", "models": [ "gemini-2.5-flash", - "llama-3.3-70b-versatile", + "llama-3.3-70b-versatile", "llama-3.3-70b", "tngtech/deepseek-r1t2-chimera:free", "openai/gpt-oss-120b", @@ -34,7 +41,7 @@ class Ayle(Provider): required_auth = False AVAILABLE_MODELS = [ "gemini-2.5-flash", - "llama-3.3-70b-versatile", + "llama-3.3-70b-versatile", "llama-3.3-70b", "tngtech/deepseek-r1t2-chimera:free", "openai/gpt-oss-120b", @@ -49,12 +56,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 4000, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "gemini-2.5-flash", system_prompt: str = "You are a friendly, helpful AI assistant.", temperature: float = 0.5, @@ -65,7 +72,7 @@ def __init__( """Initializes the Ayle client.""" if model not in self.AVAILABLE_MODELS: raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}") - + self.session = Session() # Use curl_cffi Session self.is_conversation = is_conversation self.max_tokens_to_sample = max_tokens @@ -77,10 +84,10 @@ def __init__( self.presence_penalty = presence_penalty self.frequency_penalty = frequency_penalty self.top_p = top_p - + # Initialize LitAgent for user agent generation self.agent = LitAgent() - + self.headers = { "accept": "*/*", "accept-language": "en-US,en;q=0.9", @@ -89,7 +96,7 @@ def __init__( "referer": "https://ayle.chat/", "user-agent": self.agent.random(), } - + self.session.headers.update(self.headers) self.session.proxies = proxies # Assign proxies directly self.session.cookies.update({"session": uuid.uuid4().hex}) @@ -98,7 +105,7 @@ def __init__( method for method in dir(Optimizers) if callable(getattr(Optimizers, method)) and not method.startswith("__") ) - + Conversation.intro = ( AwesomePrompts().get_act( act, raise_not_found=True, default=None, case_insensitive=True @@ -106,7 +113,7 @@ def __init__( if act else intro or Conversation.intro ) - + self.conversation = Conversation( is_conversation, self.max_tokens_to_sample, filepath, update_file ) @@ -124,12 +131,12 @@ def _get_provider_from_model(self, model: str) -> str: for provider, config in MODEL_CONFIGS.items(): if model in config["models"]: return provider - + available_models = [] for provider, config in MODEL_CONFIGS.items(): for model_name in config["models"]: available_models.append(f"{provider}/{model_name}") - + error_msg = f"Invalid model: {model}\nAvailable models: {', '.join(available_models)}" raise ValueError(error_msg) @@ -140,13 +147,13 @@ def _ayle_extractor(chunk: Union[str, Dict[str, Any]]) -> Optional[str]: if chunk.startswith('0:"'): try: return json.loads(chunk[2:]) - except: + except Exception: return None elif isinstance(chunk, dict): return chunk.get("choices", [{}])[0].get("delta", {}).get("content") return None - def _make_request(self, payload: Dict[str, Any]) -> Response: # Change type hint to Response + def _make_request(self, payload: Dict[str, Any]) -> CurlResponse: # Change type hint to Response """Make the API request with proper error handling.""" try: response = self.session.post( @@ -174,9 +181,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator[Any, None, None]]: + **kwargs: Any, + ) -> Response: """Sends a prompt to the API and returns the response.""" conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: @@ -198,7 +206,7 @@ def ask( yield_raw_on_error=False, raw=raw ) - + if stream: return self._ask_stream(prompt, processed_stream, raw) else: @@ -239,7 +247,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, ) -> Union[str, Generator[str, None, None]]: @@ -260,16 +268,19 @@ def for_non_stream(): return self.get_message(result) return for_stream() if stream else for_non_stream() - def get_message(self, response: Union[Dict[str, Any], str]) -> str: - if isinstance(response, dict): - text = response.get("text", "") - else: + def get_message(self, response: Response) -> str: + if not isinstance(response, dict): text = str(response) + else: + text = response.get("text", "") return text.replace('\\\\', '\\').replace('\\"', '"') if __name__ == "__main__": from rich import print ai = Ayle(model="gemini-2.5-flash") response = ai.chat("tell me a joke", stream=True, raw=False) - for chunk in response: - print(chunk, end='', flush=True) + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end='', flush=True) + else: + print(response) diff --git a/webscout/Provider/ChatSandbox.py b/webscout/Provider/ChatSandbox.py index f9344810..e98857c5 100644 --- a/webscout/Provider/ChatSandbox.py +++ b/webscout/Provider/ChatSandbox.py @@ -1,18 +1,13 @@ -from typing import Optional, Union, Any, Dict, Generator, List -from uuid import uuid4 import json -import re -import random +from typing import Any, Dict, Generator, Optional, Union + from curl_cffi import CurlError from curl_cffi.requests import Session -from webscout.AIutel import sanitize_stream -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts -from webscout.AIbase import Provider from webscout import exceptions -from webscout.litagent import LitAgent +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream + class ChatSandbox(Provider): """ @@ -33,16 +28,16 @@ class ChatSandbox(Provider): """ required_auth = False AVAILABLE_MODELS = [ - "openai", - "openai-gpt-4o", - "openai-o1-mini", - "deepseek", - "deepseek-r1", + "openai", + "openai-gpt-4o", + "openai-o1-mini", + "deepseek", + "deepseek-r1", "deepseek-r1-full", - "gemini", + "gemini", "gemini-thinking", - "mistral", - "mistral-large", + "mistral", + "mistral-large", "gemma-3", "llama" ] @@ -54,12 +49,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 600, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, ): """ Initializes the ChatSandbox API with given parameters. @@ -96,7 +91,7 @@ def __init__( 'origin': 'https://chatsandbox.com', 'referer': f'https://chatsandbox.com/chat/{self.model}', } - + # Update curl_cffi session headers and proxies self.session.headers.update(self.headers) self.session.proxies = proxies @@ -140,9 +135,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: """ Sends a prompt to the ChatSandbox API and returns the response. @@ -185,7 +181,7 @@ def for_stream(): raise exceptions.FailedToGenerateResponseError( f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}" ) - + streaming_response = "" # Use sanitize_stream with the custom extractor processed_stream = sanitize_stream( @@ -224,9 +220,9 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> str: + ) -> Union[str, Generator[str, None, None]]: """ Generates a response from the ChatSandbox API. @@ -262,19 +258,19 @@ def for_stream(): ) ) - def get_message(self, response: Dict[str, Any]) -> str: + def get_message(self, response: Response) -> str: """ Extract the message from the API response. - + Args: - response (Dict[str, Any]): The API response. - + response (Response): The API response. + Returns: str: The extracted message. """ if not isinstance(response, dict): return str(response) - + raw_text = response.get("text", "") # Try to parse as JSON @@ -308,8 +304,12 @@ def get_message(self, response: Dict[str, Any]) -> str: try: test_ai = ChatSandbox(model=model, timeout=60) response = test_ai.chat("Say 'Hello' in one word") - response_text = response - + + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + response_text = "".join(list(response)) + else: + response_text = str(response) + if response_text and len(response_text.strip()) > 0: status = "✓" display_text = response_text.strip()[:50].replace('\n', ' ') + ("..." if len(response_text.strip()) > 50 else "") @@ -318,4 +318,4 @@ def get_message(self, response: Dict[str, Any]) -> str: display_text = "Empty or invalid response" print(f"{model:<50} {status:<10} {display_text}") except Exception as e: - print(f"{model:<50} {'✗':<10} {str(e)}") \ No newline at end of file + print(f"{model:<50} {'✗':<10} {str(e)}") diff --git a/webscout/Provider/ClaudeOnline.py b/webscout/Provider/ClaudeOnline.py index 94b8ddb3..107e25fc 100644 --- a/webscout/Provider/ClaudeOnline.py +++ b/webscout/Provider/ClaudeOnline.py @@ -4,7 +4,7 @@ from curl_cffi.requests import Session from webscout import exceptions -from webscout.AIbase import Provider +from webscout.AIbase import Provider, Response from webscout.AIutel import AwesomePrompts, Conversation, Optimizers from webscout.litagent import LitAgent @@ -24,12 +24,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 2049, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, system_prompt: str = "You are a helpful assistant.", model: str = "claude-online" ): @@ -196,9 +196,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: """ Send a chat message and get response. @@ -278,7 +279,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, ) -> Union[str, Generator[str, None, None]]: """ @@ -309,7 +310,7 @@ def for_non_stream_chat(): return for_stream_chat() if stream else for_non_stream_chat() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """ Extract message from response. @@ -319,8 +320,9 @@ def get_message(self, response: dict) -> str: Returns: Message content """ - assert isinstance(response, dict), "Response should be of dict data-type only" - return response["text"] + if not isinstance(response, dict): + return str(response) + return response.get("text", "") if __name__ == "__main__": @@ -335,13 +337,13 @@ def get_message(self, response: dict) -> str: if limits['limit'] > 0: response = ai.chat("Say 'Hello World' in one word", stream=False) - if response and len(response.strip()) > 0: + if isinstance(response, str) and response.strip(): status = "✓" clean_text = response.strip().encode('utf-8', errors='ignore').decode('utf-8') display_text = clean_text[:50] + "..." if len(clean_text) > 50 else clean_text else: status = "✗" - display_text = "Empty response" + display_text = "Empty response or invalid type" print(f"{'claude-online':<50} {status:<10} {display_text}") else: print(f"{'claude-online':<50} {'✗':<10} Rate limit exceeded") diff --git a/webscout/Provider/Cohere.py b/webscout/Provider/Cohere.py index df6b96c3..efdd9cb2 100644 --- a/webscout/Provider/Cohere.py +++ b/webscout/Provider/Cohere.py @@ -1,11 +1,11 @@ -import requests import json -from typing import Union, Dict, Any, Generator -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider +from typing import Any, Generator, Optional, Union + +import requests + from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream #-----------------------------------------------Cohere-------------------------------------------- @@ -20,12 +20,12 @@ def __init__( temperature: float = 0.7, system_prompt: str = "You are helpful AI", timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, top_k: int = -1, top_p: float = 0.999, ): @@ -38,7 +38,7 @@ def __init__( model (str, optional): Model to use for generating text. Defaults to "command-r-plus". temperature (float, optional): Diversity of the generated text. Higher values produce more diverse outputs. Defaults to 0.7. - system_prompt (str, optional): A system_prompt or context to set the style or tone of the generated text. + system_prompt (str, optional): A system_prompt or context to set the style or tone of the generated text. Defaults to "You are helpful AI". timeout (int, optional): Http request timeout. Defaults to 30. intro (str, optional): Conversation introductory prompt. Defaults to None. @@ -88,9 +88,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> dict: + **kwargs: Any, + ) -> Response: """Chat with AI Args: @@ -170,7 +171,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, ) -> Union[str, Generator[str, None, None]]: @@ -208,20 +209,24 @@ def for_non_stream(): return for_stream() if stream else for_non_stream() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """Retrieves message only from response Args: - response (dict): Response generated by `self.ask` + response (Response): Response generated by `self.ask` Returns: str: Message extracted """ - assert isinstance(response, dict), "Response should be of dict data-type only" + if not isinstance(response, dict): + return str(response) return response["result"]["chatStreamEndEvent"]["response"]["text"] if __name__ == '__main__': from rich import print ai = Cohere(api_key="") response = ai.chat("tell me about india") - for chunk in response: - print(chunk, end="", flush=True) \ No newline at end of file + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end="", flush=True) + else: + print(response) diff --git a/webscout/Provider/DeepAI.py b/webscout/Provider/DeepAI.py index 16bf9b9f..ec547fe4 100644 --- a/webscout/Provider/DeepAI.py +++ b/webscout/Provider/DeepAI.py @@ -1,13 +1,12 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError import json -from typing import Any, Dict, List, Optional, Union, Iterator +from typing import Any, Dict, Generator, Iterator, List, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers from webscout.litagent import LitAgent @@ -20,8 +19,8 @@ class DeepAI(Provider): """ required_auth = True AVAILABLE_MODELS = [ - "standard", - "genius", + "standard", + "genius", "online", "supergenius", "onlinegenius", @@ -146,7 +145,7 @@ def __init__( ) self.conversation.history_offset = history_offset - def refresh_identity(self, browser: str = None): + def refresh_identity(self, browser: Optional[str] = None): """ Refreshes the browser identity fingerprint. @@ -174,8 +173,8 @@ def ask( raw: bool = False, optimizer: Optional[str] = None, conversationally: bool = False, - **kwargs - ) -> Union[Dict[str, Any], Iterator[Dict[str, Any]]]: + **kwargs: Any, + ) -> Response: """ Send a prompt to DeepAI and get the response. @@ -254,8 +253,8 @@ def chat( optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, - **kwargs - ) -> Union[str, Iterator[str]]: + **kwargs: Any, + ) -> Union[str, Generator[str, None, None]]: """ Send a chat message to DeepAI and get the response. @@ -297,12 +296,12 @@ def chat( else: return self.get_message(response) - def get_message(self, response: Union[Dict[str, Any], str]) -> str: + def get_message(self, response: Response) -> str: """ Extract the message from the response. Args: - response: Response dictionary from ask method or str if raw + response: Response obtained from ask method Returns: The message text @@ -312,7 +311,7 @@ def get_message(self, response: Union[Dict[str, Any], str]) -> str: elif isinstance(response, str): return response else: - raise ValueError(f"Unexpected response type: {type(response)}") + return str(response) @classmethod def get_models(cls) -> List[str]: @@ -346,8 +345,11 @@ def get_chat_styles(cls) -> List[str]: test_ai = DeepAI(model=model, timeout=60) response = test_ai.chat("Say 'Hello' in one word", stream=True) response_text = "" - for chunk in response: - response_text += chunk + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + response_text += chunk + else: + response_text = str(response) if response_text and len(response_text.strip()) > 0: status = "✓" @@ -359,4 +361,4 @@ def get_chat_styles(cls) -> List[str]: display_text = "Empty or invalid response" print(f"{model:<50} {status:<10} {display_text}") except Exception as e: - print(f"{model:<50} {'✗':<10} {str(e)}") \ No newline at end of file + print(f"{model:<50} {'✗':<10} {str(e)}") diff --git a/webscout/Provider/Deepinfra.py b/webscout/Provider/Deepinfra.py index b95b9a36..5ec70004 100644 --- a/webscout/Provider/Deepinfra.py +++ b/webscout/Provider/Deepinfra.py @@ -1,19 +1,22 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError import json -from typing import Any, Dict, Optional, Generator, Union -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session + from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent + class DeepInfra(Provider): """ A class to interact with the DeepInfra API with LitAgent user-agent. """ + required_auth = False + # Default models list (will be updated dynamically) AVAILABLE_MODELS = [ "moonshotai/Kimi-K2-Instruct", "moonshotai/Kimi-K2-Thinking", @@ -95,6 +98,57 @@ class DeepInfra(Provider): "allenai/olmOCR-7B-0725-FP8", ] + @classmethod + def get_models(cls, api_key: Optional[str] = None): + """Fetch available models from DeepInfra API. + + Args: + api_key (str, optional): DeepInfra API key. If not provided, returns default models. + + Returns: + list: List of available model IDs + """ + if not api_key: + return cls.AVAILABLE_MODELS + + try: + # Use a temporary curl_cffi session for this class method + temp_session = Session() + headers = { + "Content-Type": "application/json", + } + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + response = temp_session.get( + "https://api.deepinfra.com/v1/models", + headers=headers, + impersonate="chrome110", # Use impersonate for fetching + ) + + if response.status_code != 200: + return cls.AVAILABLE_MODELS + + data = response.json() + if "data" in data and isinstance(data["data"], list): + return [model["id"] for model in data["data"]] + return cls.AVAILABLE_MODELS + + except (CurlError, Exception): + # Fallback to default models list if fetching fails + return cls.AVAILABLE_MODELS + + @classmethod + def update_available_models(cls, api_key=None): + """Update the available models list from DeepInfra API""" + try: + models = cls.get_models(api_key) + if models and len(models) > 0: + cls.AVAILABLE_MODELS = models + except Exception: + # Fallback to default models list if fetching fails + pass + @staticmethod def _deepinfra_extractor(chunk: Union[str, Dict[str, Any]]) -> Optional[str]: """Extracts content from DeepInfra stream JSON objects.""" @@ -110,17 +164,20 @@ def __init__( is_conversation: bool = True, max_tokens: int = 2049, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "meta-llama/Llama-3.3-70B-Instruct-Turbo", system_prompt: str = "You are a helpful assistant.", - browser: str = "chrome" + browser: str = "chrome", ): """Initializes the DeepInfra API client.""" + # Update available models from API + self.update_available_models(api_key) + if model not in self.AVAILABLE_MODELS: raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}") @@ -171,9 +228,7 @@ def __init__( if callable(getattr(Optimizers, method)) and not method.startswith("__") ) Conversation.intro = ( - AwesomePrompts().get_act( - act, raise_not_found=True, default=None, case_insensitive=True - ) + AwesomePrompts().get_act(act, raise_not_found=True, default=None, case_insensitive=True) if act else intro or Conversation.intro ) @@ -193,10 +248,12 @@ def refresh_identity(self, browser: str = None): browser = browser or self.fingerprint.get("browser_type", "chrome") self.fingerprint = self.agent.generate_fingerprint(browser) - self.headers.update({ - "Accept": self.fingerprint["accept"], - "Accept-Language": self.fingerprint["accept_language"], - }) + self.headers.update( + { + "Accept": self.fingerprint["accept"], + "Accept-Language": self.fingerprint["accept_language"], + } + ) self.session.headers.update(self.headers) @@ -207,37 +264,38 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: """ Sends a prompt to the DeepInfra API and returns the response. - + Args: prompt: The prompt to send to the API stream: Whether to stream the response - raw: If True, returns unprocessed response chunks without any + raw: If True, returns unprocessed response chunks without any processing or sanitization. Useful for debugging or custom processing pipelines. Defaults to False. optimizer: Optional prompt optimizer name conversationally: Whether to use conversation context - + Returns: - When raw=False: Dict with 'text' key (non-streaming) or + When raw=False: Dict with 'text' key (non-streaming) or Generator yielding dicts (streaming) - When raw=True: Raw string response (non-streaming) or + When raw=True: Raw string response (non-streaming) or Generator yielding raw string chunks (streaming) - + Examples: >>> ai = DeepInfra() >>> # Get processed response >>> response = ai.ask("Hello") >>> print(response["text"]) - + >>> # Get raw response >>> raw_response = ai.ask("Hello", raw=True) >>> print(raw_response) - + >>> # Stream raw chunks >>> for chunk in ai.ask("Hello", stream=True, raw=True): ... print(chunk, end='', flush=True) @@ -257,18 +315,18 @@ def ask( {"role": "system", "content": self.system_prompt}, {"role": "user", "content": conversation_prompt}, ], - "stream": stream + "stream": stream, } def for_stream(): - streaming_text = "" + streaming_text = "" try: response = self.session.post( self.url, data=json.dumps(payload), stream=True, timeout=self.timeout, - impersonate="chrome110" + impersonate="chrome110", ) response.raise_for_status() @@ -279,13 +337,13 @@ def for_stream(): skip_markers=["[DONE]"], content_extractor=self._deepinfra_extractor, yield_raw_on_error=False, - raw=raw + raw=raw, ) for content_chunk in processed_stream: if isinstance(content_chunk, bytes): - content_chunk = content_chunk.decode('utf-8', errors='ignore') - + content_chunk = content_chunk.decode("utf-8", errors="ignore") + if raw: yield content_chunk else: @@ -294,22 +352,25 @@ def for_stream(): yield dict(text=content_chunk) except CurlError as e: - raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {str(e)}") from e + raise exceptions.FailedToGenerateResponseError( + f"Request failed (CurlError): {str(e)}" + ) from e except Exception as e: - raise exceptions.FailedToGenerateResponseError(f"Request failed ({type(e).__name__}): {str(e)}") from e + raise exceptions.FailedToGenerateResponseError( + f"Request failed ({type(e).__name__}): {str(e)}" + ) from e finally: if not raw and streaming_text: self.last_response = {"text": streaming_text} self.conversation.update_chat_history(prompt, streaming_text) - def for_non_stream(): try: response = self.session.post( self.url, data=json.dumps(payload), timeout=self.timeout, - impersonate="chrome110" + impersonate="chrome110", ) response.raise_for_status() @@ -321,9 +382,13 @@ def for_non_stream(): data=response.text, to_json=True, intro_value=None, - content_extractor=lambda chunk: chunk.get("choices", [{}])[0].get("message", {}).get("content") if isinstance(chunk, dict) else None, + content_extractor=lambda chunk: chunk.get("choices", [{}])[0] + .get("message", {}) + .get("content") + if isinstance(chunk, dict) + else None, yield_raw_on_error=False, - raw=raw + raw=raw, ) # Extract the single result content = next(processed_stream, None) @@ -336,11 +401,14 @@ def for_non_stream(): return self.last_response if not raw else content except CurlError as e: - raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {e}") from e + raise exceptions.FailedToGenerateResponseError( + f"Request failed (CurlError): {e}" + ) from e except Exception as e: - err_text = getattr(e, 'response', None) and getattr(e.response, 'text', '') - raise exceptions.FailedToGenerateResponseError(f"Request failed ({type(e).__name__}): {e} - {err_text}") from e - + err_text = getattr(e, "response", None) and getattr(e.response, "text", "") + raise exceptions.FailedToGenerateResponseError( + f"Request failed ({type(e).__name__}): {e} - {err_text}" + ) from e return for_stream() if stream else for_non_stream() @@ -348,71 +416,79 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, raw: bool = False, conversationally: bool = False, ) -> Union[str, Generator[str, None, None]]: """ Generates a chat response from the DeepInfra API. - + Args: prompt: The prompt to send to the API stream: Whether to stream the response optimizer: Optional prompt optimizer name - raw: If True, returns unprocessed response chunks without any + raw: If True, returns unprocessed response chunks without any processing or sanitization. Useful for debugging or custom processing pipelines. Defaults to False. conversationally: Whether to use conversation context - + Returns: When raw=False: Extracted message string or Generator yielding strings When raw=True: Raw response or Generator yielding raw chunks - + Examples: >>> ai = DeepInfra() >>> # Get processed response >>> response = ai.chat("Hello") >>> print(response) - + >>> # Get raw response >>> raw_response = ai.chat("Hello", raw=True) >>> print(raw_response) - + >>> # Stream raw chunks >>> for chunk in ai.chat("Hello", stream=True, raw=True): ... print(chunk, end='', flush=True) """ + def for_stream_chat(): # ask() yields dicts or strings when streaming gen = self.ask( - prompt, stream=True, raw=raw, - optimizer=optimizer, conversationally=conversationally + prompt, stream=True, raw=raw, optimizer=optimizer, conversationally=conversationally ) for response_dict in gen: if raw: yield response_dict else: - yield self.get_message(response_dict) # get_message expects dict + yield self.get_message(response_dict) # get_message expects dict def for_non_stream_chat(): # ask() returns dict or str when not streaming response_data = self.ask( - prompt, stream=False, raw=raw, - optimizer=optimizer, conversationally=conversationally + prompt, + stream=False, + raw=raw, + optimizer=optimizer, + conversationally=conversationally, ) if raw: return response_data else: - return self.get_message(response_data) # get_message expects dict + return self.get_message(response_data) # get_message expects dict return for_stream_chat() if stream else for_non_stream_chat() - def get_message(self, response: dict) -> str: - assert isinstance(response, dict), "Response should be of dict data-type only" + def get_message(self, response: Response) -> str: + if not isinstance(response, dict): + return str(response) return response["text"] + if __name__ == "__main__": ai = DeepInfra() response = ai.chat("Hello", raw=False, stream=True) - for chunk in response: - print(chunk, end="") \ No newline at end of file + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end="") + else: + print(response) diff --git a/webscout/Provider/EssentialAI.py b/webscout/Provider/EssentialAI.py index cf807064..7b213c95 100644 --- a/webscout/Provider/EssentialAI.py +++ b/webscout/Provider/EssentialAI.py @@ -1,16 +1,13 @@ -from typing import Generator, Optional, Union, Any, Dict import json -import time import random import string -from curl_cffi import CurlError +from typing import Any, Dict, Generator, Optional, Union + from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers from webscout.litagent import LitAgent @@ -27,12 +24,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 512, timeout: int = 60, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, system_prompt: str = "You are a helpful AI assistant.", model: str = "rnj-1-instruct", temperature: float = 0.2, @@ -64,11 +61,11 @@ def __init__( self.session = Session() self.session.headers.update(self.headers) self.session.proxies = proxies - + # Get initial cookies try: self.session.get(self.api_endpoint, timeout=self.timeout) - except: + except Exception: pass self.__available_optimizers = ( @@ -99,9 +96,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: if optimizer in self.__available_optimizers: @@ -111,12 +109,12 @@ def ask( else: raise Exception(f"Optimizer is not one of {self.__available_optimizers}") - session_hash = self._get_session_hash() - + self._get_session_hash() + # Gradio 5 /call pattern payload = { "data": [ - conversation_prompt, + conversation_prompt, [], # history self.system_prompt, float(self.max_tokens_to_sample), @@ -133,7 +131,7 @@ def for_stream(): call_response = self.session.post(call_url, json=payload, timeout=self.timeout) call_response.raise_for_status() event_id = call_response.json().get("event_id") - + if not event_id: raise exceptions.FailedToGenerateResponseError("Failed to get event_id") @@ -150,7 +148,8 @@ def for_stream(): last_full_text = "" for line in response.iter_lines(): - if not line: continue + if not line: + continue line_str = line.decode('utf-8') if line_str.startswith("data: "): try: @@ -169,7 +168,7 @@ def for_stream(): else: streaming_text += delta yield {"text": delta} - except: + except Exception: pass except Exception as e: @@ -190,7 +189,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, ) -> Union[str, Generator[str, None, None]]: @@ -202,11 +201,16 @@ def for_non_stream(): return self.get_message(result) if not raw else result return for_stream() if stream else for_non_stream() - def get_message(self, response: dict) -> str: - if not isinstance(response, dict): return str(response) + def get_message(self, response: Response) -> str: + if not isinstance(response, dict): + return str(response) return response.get("text", "") if __name__ == "__main__": ai = EssentialAI() - for chunk in ai.chat("Hello!", stream=True): - print(chunk, end="", flush=True) + response = ai.chat("Hello!", stream=True) + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end="", flush=True) + else: + print(response) diff --git a/webscout/Provider/ExaAI.py b/webscout/Provider/ExaAI.py index 0fd099c5..072000de 100644 --- a/webscout/Provider/ExaAI.py +++ b/webscout/Provider/ExaAI.py @@ -1,16 +1,15 @@ -from typing import Union, Any, Dict, Generator +import json +from typing import Any, Dict, Generator, Optional, Union from uuid import uuid4 + import requests -import json -import re -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent + class ExaAI(Provider): """ A class to interact with the o3minichat.exa.ai API. @@ -32,12 +31,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 600, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, # system_prompt: str = "You are a helpful assistant.", model: str = "O3-Mini", # >>> THIS FLAG IS NOT USED <<< ): @@ -68,7 +67,7 @@ def __init__( self.timeout = timeout self.last_response = {} # self.system_prompt = system_prompt - + # Initialize LitAgent for user agent generation self.agent = LitAgent() @@ -116,9 +115,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Dict[str, Any]: + **kwargs: Any, + ) -> Response: """ Sends a prompt to the o3minichat.exa.ai API and returns the response. @@ -150,7 +150,7 @@ def ask( ) # Generate a unique ID for the conversation - conversation_id = uuid4().hex[:16] + conversation_id = uuid4().hex[:16] payload = { "id": conversation_id, @@ -166,7 +166,7 @@ def for_stream(): raise exceptions.FailedToGenerateResponseError( f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}" ) - + streaming_response = "" # Use sanitize_stream with extract_regexes for Exa AI format processed_stream = sanitize_stream( @@ -177,7 +177,7 @@ def for_stream(): yield_raw_on_error=False, raw=raw ) - + for content in processed_stream: if content: if raw: @@ -186,7 +186,7 @@ def for_stream(): if isinstance(content, str): streaming_response += content yield dict(text=content) - + self.last_response = {"text": streaming_response} self.conversation.update_chat_history(prompt, streaming_response) @@ -201,7 +201,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, ) -> Union[str, Generator[str, None, None]]: @@ -248,24 +248,18 @@ def for_non_stream(): return for_stream() if stream else for_non_stream() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """ Extracts the message from the API response. Args: - response (dict): The API response. + response (Response): The API response. Returns: str: The message content. - - Examples: - >>> ai = ExaAI() - >>> response = ai.ask("Tell me a joke!") - >>> message = ai.get_message(response) - >>> print(message) - 'Why did the scarecrow win an award? Because he was outstanding in his field!' """ - assert isinstance(response, dict), "Response should be of dict data-type only" + if not isinstance(response, dict): + return str(response) formatted_text = response["text"].replace('\\n', '\n').replace('\\n\\n', '\n\n') return formatted_text @@ -273,5 +267,8 @@ def get_message(self, response: dict) -> str: from rich import print ai = ExaAI(timeout=5000) response = ai.chat("Tell me about HelpingAI", stream=True) - for chunk in response: - print(chunk, end="", flush=True) \ No newline at end of file + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end="", flush=True) + else: + print(response) diff --git a/webscout/Provider/Gemini.py b/webscout/Provider/Gemini.py index d51e3568..d9cf23f6 100644 --- a/webscout/Provider/Gemini.py +++ b/webscout/Provider/Gemini.py @@ -1,12 +1,12 @@ -from os import path -from json import dumps import warnings -from typing import Dict, Union, Generator +from json import dumps +from os import path +from typing import Any, Dict, Generator, Optional, Union +from ..AIbase import Provider, Response # Import internal modules and dependencies -from ..AIutel import Optimizers, Conversation, sanitize_stream -from ..AIbase import Provider +from ..AIutel import Conversation, Optimizers from ..Bard import Chatbot, Model warnings.simplefilter("ignore", category=UserWarning) @@ -19,7 +19,6 @@ "gemini-3-pro": Model.G_3_PRO, "flash-2.5": Model.G_2_5_FLASH, "pro": Model.G_2_5_PRO, - "unspecified": Model.UNSPECIFIED, } # List of available models (friendly names) @@ -80,9 +79,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> dict: + **kwargs: Any, + ) -> Response: """Chat with AI. Args: @@ -124,7 +124,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, ) -> Union[str, Generator[str, None, None]]: @@ -155,17 +155,17 @@ def for_non_stream(): return for_stream() if stream else for_non_stream() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """Retrieves message content from the response. Args: - response (dict): Response generated by `self.ask`. + response (Response): Response generated by `self.ask`. Returns: str: Extracted message content. """ if not isinstance(response, dict): - raise TypeError("Response should be of type dict") + return str(response) return response["content"] def reset(self): diff --git a/webscout/Provider/GithubChat.py b/webscout/Provider/GithubChat.py index b6bd1826..f0660657 100644 --- a/webscout/Provider/GithubChat.py +++ b/webscout/Provider/GithubChat.py @@ -1,15 +1,21 @@ -from curl_cffi import CurlError -from curl_cffi.requests import Session import json import time -from typing import Any, Dict, Optional, Union, Generator -from webscout.AIutel import Conversation -from webscout.AIutel import Optimizers -from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session + from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) from webscout.litagent import LitAgent + class GithubChat(Provider): """ A class to interact with the GitHub Copilot Chat API. @@ -21,8 +27,8 @@ class GithubChat(Provider): "gpt-4o", "gpt-5", "gpt-5-mini", - "o3-mini", - "o1", + "o3-mini", + "o1", "claude-3.5-sonnet", "claude-3.7-sonnet", "claude-3.7-sonnet-thought", @@ -34,34 +40,34 @@ class GithubChat(Provider): "o4-mini" ] - + def __init__( self, is_conversation: bool = True, max_tokens: int = 2000, timeout: int = 60, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "gpt-4o", cookie_path: str = "cookies.json" ): """Initialize the GithubChat client.""" if model not in self.AVAILABLE_MODELS: raise ValueError(f"Invalid model: {model}. Choose from: {', '.join(self.AVAILABLE_MODELS)}") - + self.url = "https://github.com/copilot" self.api_url = "https://api.individual.githubcopilot.com" self.cookie_path = cookie_path self.session = Session() # Use curl_cffi Session self.session.proxies.update(proxies) - + # Load cookies for authentication self.cookies = self.load_cookies() - + # Set up headers for all requests self.headers = { "Content-Type": "application/json", @@ -78,27 +84,27 @@ def __init__( "Sec-Fetch-Mode": "cors", "Sec-Fetch-Site": "same-origin", } - + # Apply cookies to session if self.cookies: self.session.cookies.update(self.cookies) - + # Set default model self.model = model - + # Provider settings self.is_conversation = is_conversation self.max_tokens_to_sample = max_tokens self.timeout = timeout self.last_response = {} - + # Available optimizers self.__available_optimizers = ( method for method in dir(Optimizers) if callable(getattr(Optimizers, method)) and not method.startswith("__") ) - + # Set up conversation Conversation.intro = ( AwesomePrompts().get_act( @@ -112,7 +118,7 @@ def __init__( is_conversation, self.max_tokens_to_sample, filepath, update_file ) self.conversation.history_offset = history_offset - + # Store conversation data self._conversation_id = None self._access_token = None @@ -122,7 +128,7 @@ def load_cookies(self): try: with open(self.cookie_path, 'r') as f: cookies_data = json.load(f) - + # Convert the cookie list to a dictionary format for requests cookies = {} for cookie in cookies_data: @@ -131,7 +137,7 @@ def load_cookies(self): # Check if the cookie hasn't expired if 'expirationDate' not in cookie or cookie['expirationDate'] > time.time(): cookies[cookie['name']] = cookie['value'] - + return cookies except Exception: return {} @@ -140,27 +146,27 @@ def get_access_token(self): """Get GitHub Copilot access token.""" if self._access_token: return self._access_token - + url = "https://github.com/github-copilot/chat/token" - + try: response = self.session.post(url, headers=self.headers) - + if response.status_code == 401: raise exceptions.AuthenticationError("Authentication failed. Please check your cookies.") - + if response.status_code != 200: raise exceptions.FailedToGenerateResponseError(f"Failed to get access token: {response.status_code}") - + data = response.json() self._access_token = data.get("token") - + if not self._access_token: raise exceptions.FailedToGenerateResponseError("Failed to extract access token from response") - + return self._access_token - - except: + + except Exception: pass @staticmethod @@ -174,52 +180,53 @@ def create_conversation(self): """Create a new conversation with GitHub Copilot.""" if self._conversation_id: return self._conversation_id - + access_token = self.get_access_token() url = f"{self.api_url}/github/chat/threads" - + headers = self.headers.copy() headers["Authorization"] = f"GitHub-Bearer {access_token}" - + try: response = self.session.post( url, headers=headers, impersonate="chrome120" # Add impersonate ) - + if response.status_code == 401: # Token might be expired, try refreshing self._access_token = None access_token = self.get_access_token() headers["Authorization"] = f"GitHub-Bearer {access_token}" response = self.session.post(url, headers=headers) - + # Check status after potential retry response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx) if response.status_code not in [200, 201]: raise exceptions.FailedToGenerateResponseError(f"Failed to create conversation: {response.status_code}") - + data = response.json() self._conversation_id = data.get("thread_id") - + if not self._conversation_id: raise exceptions.FailedToGenerateResponseError("Failed to extract conversation ID from response") - + return self._conversation_id except (CurlError, exceptions.FailedToGenerateResponseError, Exception) as e: # Catch CurlError and others raise exceptions.FailedToGenerateResponseError(f"Failed to create conversation: {str(e)}") - + def ask( self, prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: """Send a message to the GitHub Copilot Chat API""" - + # Apply optimizers if specified conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: @@ -229,21 +236,21 @@ def ask( ) else: raise Exception(f"Optimizer is not one of {self.__available_optimizers}") - + # Make sure we have a conversation ID try: conversation_id = self.create_conversation() except exceptions.FailedToGenerateResponseError as e: raise exceptions.FailedToGenerateResponseError(f"Failed to create conversation: {e}") - + access_token = self.get_access_token() - + url = f"{self.api_url}/github/chat/threads/{conversation_id}/messages" - + # Update headers for this specific request headers = self.headers.copy() headers["Authorization"] = f"GitHub-Bearer {access_token}" - + # Prepare the request payload request_data = { "content": conversation_prompt, @@ -257,19 +264,19 @@ def ask( "model": self.model, "mode": "immersive" } - + streaming_text = "" # Initialize for history update def for_stream(): nonlocal streaming_text # Allow modification of outer scope variable try: response = self.session.post( - url, + url, json=request_data, headers=headers, # Use updated headers with Authorization stream=True, timeout=self.timeout ) - + if response.status_code == 401: # Token might be expired, try refreshing self._access_token = None @@ -282,7 +289,7 @@ def for_stream(): stream=True, timeout=self.timeout ) - + # If still not successful, raise exception response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx) @@ -305,14 +312,14 @@ def for_stream(): streaming_text += content_chunk resp = {"text": content_chunk} yield resp if not raw else content_chunk - + except Exception as e: if isinstance(e, CurlError): # Check for CurlError if hasattr(e, 'response') and e.response is not None: - status_code = e.response.status_code + status_code = e.response.status_code if status_code == 401: raise exceptions.AuthenticationError("Authentication failed. Please check your cookies.") - + # If anything else fails raise exceptions.FailedToGenerateResponseError(f"Request failed: {str(e)}") finally: @@ -335,39 +342,43 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[str, Generator]: + ) -> Union[str, Generator[str, None, None]]: """Generate a response to a prompt""" def for_stream(): for response in self.ask( prompt, True, optimizer=optimizer, conversationally=conversationally ): yield self.get_message(response) - + def for_non_stream(): return self.get_message( self.ask( prompt, False, optimizer=optimizer, conversationally=conversationally ) ) - + return for_stream() if stream else for_non_stream() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """Extract message text from response""" - assert isinstance(response, dict), "Response should be of dict data-type only" + if not isinstance(response, dict): + return str(response) return response.get("text", "") if __name__ == "__main__": # Simple test code from rich import print - + try: - ai = GithubChat("cookies.json") + ai = GithubChat(cookie_path="cookies.json") response = ai.chat("Python code to count r in strawberry", stream=True) - for chunk in response: - print(chunk, end="", flush=True) + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end="", flush=True) + else: + print(response) print() except Exception as e: print(f"An error occurred: {e}") diff --git a/webscout/Provider/Gradient.py b/webscout/Provider/Gradient.py index e5e958f8..68ac766c 100644 --- a/webscout/Provider/Gradient.py +++ b/webscout/Provider/Gradient.py @@ -3,28 +3,29 @@ Reverse engineered from https://chat.gradient.network/ """ +from typing import Any, Dict, Generator, Optional, Union + import requests -from typing import Optional, Generator, Dict, Any, Union -from webscout.AIutel import Optimizers, Conversation, AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream class Gradient(Provider): """ Provider for Gradient Network chat API Supports real-time streaming responses from distributed GPU clusters - + Note: GPT OSS 120B works on "nvidia" cluster, Qwen3 235B works on "hybrid" cluster """ - + required_auth = False AVAILABLE_MODELS = [ "GPT OSS 120B", "Qwen3 235B", ] - + # Model to cluster mapping MODEL_CLUSTERS = { "GPT OSS 120B": "nvidia", @@ -37,14 +38,14 @@ def __init__( is_conversation: bool = True, max_tokens: int = 2049, timeout: int = 60, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, system_prompt: str = "You are a helpful assistant.", - cluster_mode: str = None, # Auto-detected based on model if None + cluster_mode: Optional[str] = None, # Auto-detected based on model if None enable_thinking: bool = True, ): # Normalize model name (convert dashes to spaces) @@ -62,11 +63,11 @@ def __init__( self.cluster_mode = cluster_mode or self.MODEL_CLUSTERS.get(model, "nvidia") self.enable_thinking = enable_thinking self.last_response = {} - + self.session = requests.Session() if proxies: self.session.proxies = proxies - + # Headers matching the working curl request self.headers = { "accept": "*/*", @@ -109,14 +110,14 @@ def __init__( def _gradient_extractor(chunk: Union[str, Dict[str, Any]]) -> Optional[str]: """ Extracts content from Gradient API stream response. - + The API returns JSON objects like: {"type": "reply", "data": {"role": "assistant", "content": "text"}} {"type": "reply", "data": {"role": "assistant", "reasoningContent": "text"}} - + Args: chunk: Parsed JSON dict from the stream - + Returns: Extracted content string or None """ @@ -135,9 +136,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: if optimizer in self.__available_optimizers: @@ -203,7 +205,7 @@ def for_non_stream(): full_response = "" for chunk in for_stream(): full_response += self.get_message(chunk) if not raw else chunk - + self.last_response = {"text": full_response} self.conversation.update_chat_history(prompt, full_response) return self.last_response if not raw else full_response @@ -217,7 +219,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, ) -> Union[str, Generator[str, None, None]]: def for_stream_chat(): @@ -252,8 +254,11 @@ def get_message(self, response: dict) -> str: test_ai = Gradient(model=model, timeout=120) response = test_ai.chat("Say 'Hello' in one word", stream=True) response_text = "" - for chunk in response: - response_text += chunk + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + response_text += chunk + else: + response_text = str(response) if response_text and len(response_text.strip()) > 0: status = "✓" diff --git a/webscout/Provider/Groq.py b/webscout/Provider/Groq.py index 48999939..7ea1a57e 100644 --- a/webscout/Provider/Groq.py +++ b/webscout/Provider/Groq.py @@ -1,17 +1,20 @@ -from typing import Any, AsyncGenerator, Dict, Optional, Callable, List, Union - -import httpx import json +from typing import Any, Callable, Dict, Generator, List, Optional, Union + +from curl_cffi import CurlError # Import curl_cffi for improved request handling from curl_cffi.requests import Session -from curl_cffi import CurlError -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) + class GROQ(Provider): """ @@ -46,20 +49,20 @@ class GROQ(Provider): "llama-3.2-90b-vision-preview", "mixtral-8x7b-32768" ] - + @classmethod - def get_models(cls, api_key: str = None): + def get_models(cls, api_key: Optional[str] = None): """Fetch available models from Groq API. - + Args: api_key (str, optional): Groq API key. If not provided, returns default models. - + Returns: list: List of available model IDs """ if not api_key: return cls.AVAILABLE_MODELS - + try: # Use a temporary curl_cffi session for this class method temp_session = Session() @@ -67,21 +70,21 @@ def get_models(cls, api_key: str = None): "Content-Type": "application/json", "Authorization": f"Bearer {api_key}", } - + response = temp_session.get( "https://api.groq.com/openai/v1/models", headers=headers, impersonate="chrome110" # Use impersonate for fetching ) - + if response.status_code != 200: return cls.AVAILABLE_MODELS - + data = response.json() if "data" in data and isinstance(data["data"], list): return [model["id"] for model in data["data"]] return cls.AVAILABLE_MODELS - + except (CurlError, Exception): # Fallback to default models list if fetching fails return cls.AVAILABLE_MODELS @@ -97,12 +100,12 @@ def __init__( top_p: float = 1, model: str = "mixtral-8x7b-32768", timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, system_prompt: Optional[str] = None, ): """Instantiates GROQ @@ -127,7 +130,7 @@ def __init__( """ # Update available models from API self.update_available_models(api_key) - + # Validate model after updating available models if model not in self.AVAILABLE_MODELS: raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}") @@ -142,7 +145,7 @@ def __init__( self.presence_penalty = presence_penalty self.frequency_penalty = frequency_penalty self.top_p = top_p - self.chat_endpoint = "https://api.groq.com/openai/v1/chat/completions" + self.chat_endpoint = "https://api.groq.com/openai/v1/chat/completions" self.stream_chunk_size = 64 self.timeout = timeout self.last_response = {} @@ -158,10 +161,10 @@ def __init__( for method in dir(Optimizers) if callable(getattr(Optimizers, method)) and not method.startswith("__") ) - + # Update curl_cffi session headers self.session.headers.update(self.headers) - + # Set up conversation Conversation.intro = ( AwesomePrompts().get_act( @@ -174,10 +177,10 @@ def __init__( is_conversation, self.max_tokens_to_sample, filepath, update_file ) self.conversation.history_offset = history_offset - + # Set proxies for curl_cffi session self.session.proxies = proxies - + @staticmethod def _groq_extractor(chunk: Union[str, Dict[str, Any]]) -> Optional[Dict]: """Extracts the 'delta' object from Groq stream JSON chunks.""" @@ -211,10 +214,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, tools: Optional[List[Dict[str, Any]]] = None, # Add tools parameter - ) -> dict: + ) -> Response: """Chat with AI Args: @@ -258,9 +261,9 @@ def ask( def for_stream(): try: response = self.session.post( - self.chat_endpoint, - json=payload, - stream=True, + self.chat_endpoint, + json=payload, + stream=True, timeout=self.timeout, impersonate="chrome110" # Use impersonate for better compatibility ) @@ -301,15 +304,19 @@ def for_stream(): raise exceptions.FailedToGenerateResponseError(f"Error: {str(e)}") # Handle tool calls if any - if 'tool_calls' in self.last_response.get('choices', [{}])[0].get('message', {}): - tool_calls = self.last_response['choices'][0]['message']['tool_calls'] + first_choice = self.last_response.get('choices', [{}])[0] + message = first_choice.get('message', {}) + if 'tool_calls' in message: + tool_calls = message.get('tool_calls', []) for tool_call in tool_calls: + if not isinstance(tool_call, dict): + continue function_name = tool_call.get('function', {}).get('name') arguments = json.loads(tool_call.get('function', {}).get('arguments', "{}")) if function_name in self.available_functions: tool_response = self.available_functions[function_name](**arguments) messages.append({ - "tool_call_id": tool_call['id'], + "tool_call_id": tool_call.get('id'), "role": "tool", "name": function_name, "content": tool_response @@ -318,8 +325,8 @@ def for_stream(): # Make a second call to get the final response try: second_response = self.session.post( - self.chat_endpoint, - json=payload, + self.chat_endpoint, + json=payload, timeout=self.timeout, impersonate="chrome110" # Use impersonate for better compatibility ) @@ -341,9 +348,9 @@ def for_stream(): def for_non_stream(): try: response = self.session.post( - self.chat_endpoint, - json=payload, - stream=False, + self.chat_endpoint, + json=payload, + stream=False, timeout=self.timeout, impersonate="chrome110" # Use impersonate for better compatibility ) @@ -354,7 +361,7 @@ def for_non_stream(): # Removed response.reason_phrase f"Failed to generate response - ({response.status_code}) - {response.text}" ) - + response_text = response.text # Get raw text # Use sanitize_stream to parse the non-streaming JSON response @@ -367,7 +374,7 @@ def for_non_stream(): yield_raw_on_error=False, raw=raw ) - + # Extract the single result (the parsed JSON dictionary) resp = next(processed_stream, None) if raw: @@ -386,15 +393,19 @@ def for_non_stream(): raise exceptions.FailedToGenerateResponseError(f"Error: {str(e)}") # Handle tool calls if any - if 'tool_calls' in resp.get('choices', [{}])[0].get('message', {}): - tool_calls = resp['choices'][0]['message']['tool_calls'] + first_choice = resp.get('choices', [{}])[0] + message = first_choice.get('message', {}) + if 'tool_calls' in message: + tool_calls = message.get('tool_calls', []) for tool_call in tool_calls: + if not isinstance(tool_call, dict): + continue function_name = tool_call.get('function', {}).get('name') arguments = json.loads(tool_call.get('function', {}).get('arguments', "{}")) if function_name in self.available_functions: tool_response = self.available_functions[function_name](**arguments) messages.append({ - "tool_call_id": tool_call['id'], + "tool_call_id": tool_call.get('id'), "role": "tool", "name": function_name, "content": tool_response @@ -403,8 +414,8 @@ def for_non_stream(): # Make a second call to get the final response try: second_response = self.session.post( - self.chat_endpoint, - json=payload, + self.chat_endpoint, + json=payload, timeout=self.timeout, impersonate="chrome110" # Use impersonate for better compatibility ) @@ -431,10 +442,10 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, tools: Optional[List[Dict[str, Any]]] = None, - ) -> str: + ) -> Union[str, Generator[str, None, None]]: """Generate response `str` Args: prompt (str): Prompt to be send. @@ -465,16 +476,17 @@ def for_non_stream(): return for_stream() if stream else for_non_stream() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """Retrieves message only from response Args: - response (dict): Response generated by `self.ask` + response (Response): Response generated by `self.ask` Returns: str: Message extracted """ - assert isinstance(response, dict), "Response should be of dict data-type only" + if not isinstance(response, dict): + return str(response) try: # Check delta first for streaming if response.get("choices") and response["choices"][0].get("delta") and response["choices"][0]["delta"].get("content"): @@ -493,4 +505,4 @@ def get_message(self, response: dict) -> str: groq = GROQ(api_key=api_key, model="compound-beta") prompt = "What is the capital of France?" response = groq.chat(prompt) - print(response) \ No newline at end of file + print(response) diff --git a/webscout/Provider/HadadXYZ.py b/webscout/Provider/HadadXYZ.py index 9c188c3c..0c9a25be 100644 --- a/webscout/Provider/HadadXYZ.py +++ b/webscout/Provider/HadadXYZ.py @@ -12,9 +12,9 @@ from curl_cffi import CurlError from curl_cffi.requests import Session +from webscout import exceptions from webscout.AIbase import Provider from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream -from webscout import exceptions class _DeltaExtractor: @@ -295,4 +295,4 @@ def get_message(self, response: Dict[str, Any]) -> str: ai = HadadXYZ(model="deepseek-ai/deepseek-r1-0528") for chunk in ai.chat("how many r in strawberry", stream=True): - print(chunk, end="", flush=True) \ No newline at end of file + print(chunk, end="", flush=True) diff --git a/webscout/Provider/HeckAI.py b/webscout/Provider/HeckAI.py index c16cb192..ed5cb97c 100644 --- a/webscout/Provider/HeckAI.py +++ b/webscout/Provider/HeckAI.py @@ -1,16 +1,16 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError import json import uuid -from typing import Any, Dict, Optional, Generator, Union +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent + class HeckAI(Provider): """ Provides an interface to interact with the HeckAI API using a LitAgent user-agent. @@ -55,12 +55,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 2049, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "google/gemini-2.5-flash-preview", language: str = "English" ): @@ -85,11 +85,11 @@ def __init__( """ if model not in self.AVAILABLE_MODELS: raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}") - + self.url = "https://api.heckai.weight-wave.com/api/ha/v1/chat" self.session_id = str(uuid.uuid4()) self.language = language - + # Use LitAgent (keep if needed for other headers or logic) self.headers = { 'Content-Type': 'application/json', @@ -97,7 +97,7 @@ def __init__( 'Referer': 'https://heck.ai/', # Keep Referer 'User-Agent': LitAgent().random(), # Use random user agent } - + # Initialize curl_cffi Session self.session = Session() # Update curl_cffi session headers and proxies @@ -135,9 +135,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: """ Sends a prompt to the HeckAI API and returns the response. @@ -174,7 +175,7 @@ def ask( "imgUrls": [], "superSmartMode": False # Added based on API request data } - + # Store this message as previous for next request self.previous_question = conversation_prompt @@ -182,9 +183,9 @@ def for_stream(): streaming_text = "" # Initialize outside try block try: response = self.session.post( - self.url, - data=json.dumps(payload), - stream=True, + self.url, + data=json.dumps(payload), + stream=True, timeout=self.timeout, impersonate="chrome110" ) @@ -213,7 +214,7 @@ def for_stream(): if content_chunk and isinstance(content_chunk, str): streaming_text += content_chunk yield dict(text=content_chunk) - + # Only update history if we received a valid response if streaming_text: self.previous_answer = streaming_text @@ -264,21 +265,21 @@ def fix_encoding(text): if isinstance(text, dict) and "text" in text: try: text["text"] = text["text"].encode("latin1").decode("utf-8") - return text.replace('\\\\', '\\').replace('\\"', '"') # Handle escaped backslashes - except (UnicodeError, AttributeError) as e: - return text.replace('\\\\', '\\').replace('\\"', '"') # Handle escaped backslashes + return text.replace('\\\\', '\\').replace('\\"', '"') # Handle escaped backslashes + except (UnicodeError, AttributeError): + return text.replace('\\\\', '\\').replace('\\"', '"') # Handle escaped backslashes elif isinstance(text, str): try: return text.encode("latin1").decode("utf-8") - except (UnicodeError, AttributeError) as e: - return text.replace('\\\\', '\\').replace('\\"', '"') # Handle escaped backslashes - return text.replace('\\\\', '\\').replace('\\"', '"') # Handle escaped backslashes + except (UnicodeError, AttributeError): + return text.replace('\\\\', '\\').replace('\\"', '"') # Handle escaped backslashes + return text.replace('\\\\', '\\').replace('\\"', '"') # Handle escaped backslashes def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, ) -> Union[str, Generator[str, None, None]]: @@ -305,7 +306,7 @@ def for_stream_chat(): yield response else: yield self.get_message(response) - + def for_non_stream_chat(): # ask() returns dict or str when not streaming response_data = self.ask( @@ -315,25 +316,22 @@ def for_non_stream_chat(): if raw: return response_data if isinstance(response_data, str) else str(response_data) return self.get_message(response_data) # get_message expects dict - + return for_stream_chat() if stream else for_non_stream_chat() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """ Extracts the message text from the API response. Args: - response (dict): The API response dictionary. + response (Response): The API response dictionary. Returns: str: The extracted message text. Returns an empty string if not found. - - Raises: - TypeError: If the response is not a dictionary. """ # Validate response format if not isinstance(response, dict): - raise TypeError(f"Expected dict response, got {type(response).__name__}") + return str(response) # Handle missing text key gracefully if "text" not in response: @@ -347,38 +345,11 @@ def get_message(self, response: dict) -> str: return text.replace('\\\\', '\\').replace('\\"', '"') if __name__ == "__main__": - # # Ensure curl_cffi is installed - # print("-" * 80) - # print(f"{'Model':<50} {'Status':<10} {'Response'}") - # print("-" * 80) - - # for model in HeckAI.AVAILABLE_MODELS: - # try: - # test_ai = HeckAI(model=model, timeout=60) - # # Use non-streaming mode first to avoid potential streaming issues - # try: - # response_text = test_ai.chat("Say 'Hello' in one word", stream=False) - # print(f"\r{model:<50} {'✓':<10} {response_text.strip()[:50]}") - # except Exception as e1: - # # Fall back to streaming if non-streaming fails - # print(f"\r{model:<50} {'Testing stream...':<10}", end="", flush=True) - # response = test_ai.chat("Say 'Hello' in one word", stream=True) - # response_text = "" - # for chunk in response: - # if chunk and isinstance(chunk, str): - # response_text += chunk - - # if response_text and len(response_text.strip()) > 0: - # status = "✓" - # # Truncate response if too long - # display_text = response_text.strip()[:50] + "..." if len(response_text.strip()) > 50 else response_text.strip() - # print(f"\r{model:<50} {status:<10} {display_text}") - # else: - # raise ValueError("Empty or invalid response") - # except Exception as e: - # print(f"\r{model:<50} {'✗':<10} {str(e)}") from rich import print ai = HeckAI(model="openai/gpt-5-nano") response = ai.chat("tell me about humans", stream=True, raw=False) - for chunk in response: - print(chunk, end='', flush=True) \ No newline at end of file + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end='', flush=True) + else: + print(response) diff --git a/webscout/Provider/HuggingFace.py b/webscout/Provider/HuggingFace.py index 563800b7..1decc9b1 100644 --- a/webscout/Provider/HuggingFace.py +++ b/webscout/Provider/HuggingFace.py @@ -1,14 +1,15 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError import json -from typing import Any, Dict, Optional, Generator, Union, List -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider +from typing import Any, Dict, Generator, List, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session + from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent + class HuggingFace(Provider): """ A class to interact with the Hugging Face Router API with LitAgent user-agent. @@ -18,7 +19,7 @@ class HuggingFace(Provider): AVAILABLE_MODELS = [] @classmethod - def get_models(cls, api_key: str = None) -> List[str]: + def get_models(cls, api_key: Optional[str] = None) -> list[str]: """Fetch available text-generation models from Hugging Face.""" url = "https://router.huggingface.co/v1/models" try: @@ -26,7 +27,7 @@ def get_models(cls, api_key: str = None) -> List[str]: headers = {} if api_key: headers["Authorization"] = f"Bearer {api_key}" - + response = temp_session.get(url, headers=headers, timeout=10) if response.status_code == 200: data = response.json() @@ -38,7 +39,7 @@ def get_models(cls, api_key: str = None) -> List[str]: return cls.AVAILABLE_MODELS @classmethod - def update_available_models(cls, api_key: str = None): + def update_available_models(cls, api_key: Optional[str] = None): """Update the available models list from Hugging Face API dynamically.""" try: models = cls.get_models(api_key) @@ -53,12 +54,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 2049, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "meta-llama/Llama-3.3-70B-Instruct", system_prompt: str = "You are a helpful assistant.", temperature: float = 0.7, @@ -157,37 +158,38 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: """ Sends a prompt to the Hugging Face Router API and returns the response. - + Args: prompt: The prompt to send to the API stream: Whether to stream the response - raw: If True, returns unprocessed response chunks without any + raw: If True, returns unprocessed response chunks without any processing or sanitization. Useful for debugging or custom processing pipelines. Defaults to False. optimizer: Optional prompt optimizer name conversationally: Whether to use conversation context - + Returns: - When raw=False: Dict with 'text' key (non-streaming) or + When raw=False: Dict with 'text' key (non-streaming) or Generator yielding dicts (streaming) - When raw=True: Raw string response (non-streaming) or + When raw=True: Raw string response (non-streaming) or Generator yielding raw string chunks (streaming) - + Examples: >>> hf = HuggingFace(api_key="your-key") >>> # Get processed response >>> response = hf.ask("Hello") >>> print(response["text"]) - + >>> # Get raw response >>> raw_response = hf.ask("Hello", raw=True) >>> print(raw_response) - + >>> # Stream raw chunks >>> for chunk in hf.ask("Hello", stream=True, raw=True): ... print(chunk, end='', flush=True) @@ -214,7 +216,7 @@ def ask( } def for_stream(): - streaming_text = "" + streaming_text = "" try: response = self.session.post( self.url, @@ -237,7 +239,7 @@ def for_stream(): for content_chunk in processed_stream: if isinstance(content_chunk, bytes): content_chunk = content_chunk.decode('utf-8', errors='ignore') - + if raw: yield content_chunk else: @@ -300,7 +302,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, raw: bool = False, conversationally: bool = False, ) -> Union[str, Generator[str, None, None]]: @@ -330,14 +332,19 @@ def for_non_stream_chat(): return for_stream_chat() if stream else for_non_stream_chat() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """Retrieves message from response dict.""" - assert isinstance(response, dict), "Response should be of dict data-type only" + if not isinstance(response, dict): + return str(response) return response.get("text", "") if __name__ == "__main__": hf = HuggingFace(api_key="") models = hf.AVAILABLE_MODELS print(models) - for chunk in hf.chat("Hi!", stream=True): - print(chunk, end="", flush=True) + response = hf.chat("Hi!", stream=True) + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end="", flush=True) + else: + print(response) diff --git a/webscout/Provider/IBM.py b/webscout/Provider/IBM.py index c7c29828..ecaec0be 100644 --- a/webscout/Provider/IBM.py +++ b/webscout/Provider/IBM.py @@ -1,16 +1,22 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError import json -from typing import Any, Dict, Optional, Generator, Union -from datetime import datetime import uuid -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider +from datetime import datetime +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session + from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) from webscout.litagent import LitAgent + class IBM(Provider): """ A class to interact with the IBM Granite Playground API. @@ -54,12 +60,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 2049, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "granite-chat", system_prompt: str = "You are a helpful assistant.", browser: str = "chrome" # Note: browser fingerprinting might be less effective with impersonate @@ -150,11 +156,12 @@ def refresh_identity(self, browser: str = None): def ask( self, prompt: str, - stream: bool = False, # API supports streaming + stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: if optimizer in self.__available_optimizers: @@ -197,7 +204,7 @@ def for_stream(): timeout=self.timeout, impersonate="chrome110" # Use a common impersonation profile ) - + if response.status_code in [401, 403]: # Token expired, refresh and retry once self.get_token() @@ -208,7 +215,7 @@ def for_stream(): timeout=self.timeout, impersonate="chrome110" ) - + response.raise_for_status() # Check for HTTP errors # Use sanitize_stream @@ -255,10 +262,10 @@ def for_non_stream(): final_content += chunk_data["text"] elif isinstance(chunk_data, str): final_content += chunk_data - + if not final_content: raise exceptions.FailedToGenerateResponseError("Empty response from provider") - + self.last_response = {"text": final_content} return self.last_response if not raw else final_content @@ -272,7 +279,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, ) -> Union[str, Generator[str, None, None]]: def for_stream_chat(): @@ -294,8 +301,9 @@ def for_non_stream_chat(): return for_stream_chat() if stream else for_non_stream_chat() - def get_message(self, response: dict) -> str: - assert isinstance(response, dict), "Response should be of dict data-type only" + def get_message(self, response: Response) -> str: + if not isinstance(response, dict): + return str(response) return response["text"] if __name__ == "__main__": @@ -309,8 +317,11 @@ def get_message(self, response: dict) -> str: test_ai = IBM(model=model, timeout=60,) response = test_ai.chat("Say 'Hello' in one word", stream=True) response_text = "" - for chunk in response: - response_text += chunk + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + response_text += chunk + else: + response_text = str(response) if response_text and len(response_text.strip()) > 0: status = "✓" @@ -322,4 +333,4 @@ def get_message(self, response: dict) -> str: display_text = "Empty or invalid response" print(f"\r{model:<50} {status:<10} {display_text}") except Exception as e: - print(f"\r{model:<50} {'✗':<10} {str(e)}") \ No newline at end of file + print(f"\r{model:<50} {'✗':<10} {str(e)}") diff --git a/webscout/Provider/Jadve.py b/webscout/Provider/Jadve.py index 2a5500d2..c84cd797 100644 --- a/webscout/Provider/Jadve.py +++ b/webscout/Provider/Jadve.py @@ -1,13 +1,19 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError import re -from typing import Union, Any, Dict, Optional, Generator import secrets +from typing import Any, Dict, Generator, Optional, Union +from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers, Conversation, AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) + class JadveOpenAI(Provider): """ @@ -21,12 +27,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 600, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "gpt-5-mini", system_prompt: str = "You are a helpful AI assistant." # Note: system_prompt is not used by this API ): @@ -78,7 +84,7 @@ def __init__( "sec-gpc": "1", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 Edg/140.0.0.0" } - + # Update curl_cffi session headers and proxies self.session.headers.update(self.headers) self.session.proxies = proxies # Assign proxies directly @@ -115,11 +121,12 @@ def _jadve_extractor(chunk: Union[str, Dict[str, Any]]) -> Optional[str]: def ask( self, prompt: str, - stream: bool = False, # API supports streaming + stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[dict, Generator[dict, None, None]]: + **kwargs: Any, + ) -> Response: """ Chat with AI. @@ -157,10 +164,10 @@ def for_stream(): try: # Use curl_cffi session post with impersonate response = self.session.post( - self.api_endpoint, + self.api_endpoint, # headers are set on the session - json=payload, - stream=True, + json=payload, + stream=True, timeout=self.timeout, # proxies are set on the session impersonate="chrome120" # Use a common impersonation profile @@ -220,12 +227,12 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, ) -> Union[str, Generator[str, None, None]]: """ - Generate a chat response (string). + Generate a chat response (string). Args: prompt (str): Prompt to be sent. @@ -258,22 +265,23 @@ def for_non_stream_chat(): return for_stream_chat() if stream else for_non_stream_chat() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """ Retrieves message from the response. Args: - response (dict): Response from the ask() method. + response (Response): Response from the ask() method. Returns: str: Extracted text. """ - assert isinstance(response, dict), "Response should be of dict data-type only" + if not isinstance(response, dict): + return str(response) # Extractor handles formatting return response.get("text", "") if __name__ == "__main__": for model in JadveOpenAI.AVAILABLE_MODELS: - ai = JadveOpenAI(model=model) + ai = JadveOpenAI(model=model) response = ai.chat("hi") print(f"Model: {model}") print(response) diff --git a/webscout/Provider/K2Think.py b/webscout/Provider/K2Think.py index cc0f6527..602b63d0 100644 --- a/webscout/Provider/K2Think.py +++ b/webscout/Provider/K2Think.py @@ -1,18 +1,16 @@ import json -import os -from typing import Any, Dict, Optional, Generator, Union, List +from typing import Any, Dict, Generator, Optional, Union -from curl_cffi.requests import Session from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers from webscout.litagent import LitAgent from webscout.sanitize import sanitize_stream + class K2Think(Provider): """ A class to interact with the K2Think AI API. @@ -33,12 +31,12 @@ def __init__( top_p: float = 1, model: str = "MBZUAI-IFM/K2-Think", timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, base_url: str = "https://www.k2think.ai/api/guest/chat/completions", system_prompt: str = "You are a helpful assistant.", browser: str = "chrome" @@ -130,9 +128,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: if optimizer in self.__available_optimizers: @@ -166,17 +165,16 @@ def for_stream(): response.raise_for_status() streaming_text = "" - + # Use sanitize_stream with extract_regexes - import re answer_pattern = r'([\s\S]*?)<\/answer>' - + def content_extractor(data): """Extract 'content' field from JSON object""" if isinstance(data, dict): return data.get('content', '') return None - + for chunk in sanitize_stream( response.iter_lines(), intro_value="data:", @@ -224,7 +222,7 @@ def for_non_stream(): content = data["choices"][0].get("message", {}).get("content", "") else: content = data.get("content", "") - + # Extract using regex if needed (for reasoning models) import re answer_match = re.search(r'([\s\S]*?)<\/answer>', content) @@ -252,7 +250,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, ) -> Union[str, Generator[str, None, None]]: @@ -273,8 +271,9 @@ def for_non_stream_chat(): return for_stream_chat() if stream else for_non_stream_chat() - def get_message(self, response: dict) -> str: - assert isinstance(response, dict), "Response should be of dict data-type only" + def get_message(self, response: Response) -> str: + if not isinstance(response, dict): + return str(response) return response["text"].replace('\\n', '\n').replace('\\n\\n', '\n\n') if __name__ == "__main__": @@ -282,10 +281,13 @@ def get_message(self, response: dict) -> str: try: ai = K2Think(model="MBZUAI-IFM/K2-Think", timeout=30) response = ai.chat("What is artificial intelligence?", stream=True, raw=False) - for chunk in response: - print(chunk, end="", flush=True) + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end="", flush=True) + else: + print(response) print() except Exception as e: print(f"Error: {type(e).__name__}: {e}") import traceback - traceback.print_exc() \ No newline at end of file + traceback.print_exc() diff --git a/webscout/Provider/Koboldai.py b/webscout/Provider/Koboldai.py index 99d8a4f2..05e21e72 100644 --- a/webscout/Provider/Koboldai.py +++ b/webscout/Provider/Koboldai.py @@ -1,12 +1,12 @@ -import requests import json -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider -from webscout import exceptions -from typing import Union, Any, AsyncGenerator, Dict -import httpx +from typing import Any, Generator, Optional, Union + +import requests + +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers + + #------------------------------------------------------KOBOLDAI----------------------------------------------------------- class KOBOLDAI(Provider): required_auth = False @@ -17,12 +17,12 @@ def __init__( temperature: float = 1, top_p: float = 1, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, ): """Instantiate TGPT @@ -79,9 +79,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> dict: + **kwargs: Any, + ) -> Response: """Chat with AI Args: @@ -158,9 +159,9 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> str: + ) -> Union[str, Generator[str, None, None]]: """Generate response `str` Args: prompt (str): Prompt to be send. @@ -189,18 +190,19 @@ def for_non_stream(): return for_stream() if stream else for_non_stream() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """Retrieves message only from response Args: - response (dict): Response generated by `self.ask` + response (Response): Response generated by `self.ask` Returns: str: Message extracted """ - assert isinstance(response, dict), "Response should be of dict data-type only" - return response.get("token") + if not isinstance(response, dict): + return str(response) + return response.get("token", "") if __name__ == "__main__": koboldai = KOBOLDAI(is_conversation=True, max_tokens=600, temperature=0.7) - print(koboldai.chat("Explain quantum computing in simple terms", stream=False)) \ No newline at end of file + print(koboldai.chat("Explain quantum computing in simple terms", stream=False)) diff --git a/webscout/Provider/Netwrck.py b/webscout/Provider/Netwrck.py index 52f2150b..f4ab3efe 100644 --- a/webscout/Provider/Netwrck.py +++ b/webscout/Provider/Netwrck.py @@ -1,11 +1,15 @@ -from typing import Any, Dict, Optional, Generator, Union -from webscout.AIutel import Optimizers, Conversation, AwesomePrompts # Import sanitize_stream -from webscout.AIbase import Provider +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import CurlError # Import CurlError + +# Replace requests with curl_cffi +from curl_cffi.requests import Session # Import Session + from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers # Import sanitize_stream from webscout.litagent import LitAgent -# Replace requests with curl_cffi -from curl_cffi.requests import Session # Import Session -from curl_cffi import CurlError # Import CurlError + class Netwrck(Provider): """ @@ -54,7 +58,7 @@ def __init__( self.last_response: Dict[str, Any] = {} self.temperature = temperature self.top_p = top_p - + self.agent = LitAgent() # Keep for potential future use or other headers self.headers = { 'authority': 'netwrck.com', @@ -63,10 +67,10 @@ def __init__( 'content-type': 'application/json', 'origin': 'https://netwrck.com', 'referer': 'https://netwrck.com/', - 'user-agent': self.agent.random() + 'user-agent': self.agent.random() # Add sec-ch-ua headers if needed for impersonation consistency } - + # Update curl_cffi session headers and proxies self.session.headers.update(self.headers) self.proxies = proxies or {} @@ -77,7 +81,7 @@ def __init__( if act else intro or Conversation.intro ) - + self.conversation = Conversation(is_conversation, max_tokens, filepath, update_file) self.conversation.history_offset = history_offset self.__available_optimizers = ( @@ -101,7 +105,8 @@ def ask( raw: bool = False, # Keep raw param for interface consistency optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: """Sends a prompt to the Netwrck API and returns the response.""" if optimizer and optimizer not in self.__available_optimizers: raise exceptions.FailedToGenerateResponseError(f"Optimizer is not one of {self.__available_optimizers}") @@ -185,7 +190,7 @@ def chat( stream: bool = False, optimizer: Optional[str] = None, conversationally: bool = False, - ) -> str: + ) -> Union[str, Generator[str, None, None]]: """Generates a response from the Netwrck API.""" def for_stream_chat(): # ask() yields dicts or strings when streaming @@ -212,9 +217,10 @@ def for_non_stream_chat(): return for_stream_chat() if stream else for_non_stream_chat() - def get_message(self, response: Dict[str, Any]) -> str: + def get_message(self, response: Response) -> str: """Retrieves message only from response""" - assert isinstance(response, dict), "Response should be of dict data-type only" + if not isinstance(response, dict): + return str(response) return response["text"].replace('\\n', '\n').replace('\\n\\n', '\n\n') if __name__ == "__main__": @@ -222,20 +228,23 @@ def get_message(self, response: Dict[str, Any]) -> str: print("-" * 80) print(f"{'Model':<50} {'Status':<10} {'Response'}") print("-" * 80) - + # Test all available models working = 0 total = len(Netwrck.AVAILABLE_MODELS) - + for model in Netwrck.AVAILABLE_MODELS: try: test_ai = Netwrck(model=model, timeout=60) response = test_ai.chat("Say 'Hello' in one word", stream=True) response_text = "" - for chunk in response: - response_text += chunk - print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True) - + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + response_text += chunk + print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True) + else: + response_text = str(response) + if response_text and len(response_text.strip()) > 0: status = "✓" # Truncate response if too long diff --git a/webscout/Provider/Nvidia.py b/webscout/Provider/Nvidia.py index 1aee19fb..80730666 100644 --- a/webscout/Provider/Nvidia.py +++ b/webscout/Provider/Nvidia.py @@ -1,14 +1,15 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError import json -from typing import Any, Dict, Optional, Generator, Union, List -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider +from typing import Any, Dict, Generator, List, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session + from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent + class Nvidia(Provider): """ A class to interact with the Nvidia NIM API with LitAgent user-agent. @@ -18,7 +19,7 @@ class Nvidia(Provider): AVAILABLE_MODELS = [] @classmethod - def get_models(cls, api_key: str = None) -> List[str]: + def get_models(cls, api_key: Optional[str] = None) -> list[str]: """Fetch available models from Nvidia API.""" url = "https://integrate.api.nvidia.com/v1/models" try: @@ -26,7 +27,7 @@ def get_models(cls, api_key: str = None) -> List[str]: headers = {} if api_key: headers["Authorization"] = f"Bearer {api_key}" - + response = temp_session.get(url, headers=headers, timeout=10) if response.status_code == 200: data = response.json() @@ -37,7 +38,7 @@ def get_models(cls, api_key: str = None) -> List[str]: return cls.AVAILABLE_MODELS @classmethod - def update_available_models(cls, api_key: str = None): + def update_available_models(cls, api_key: Optional[str] = None): """Update the available models list from Nvidia API dynamically.""" try: models = cls.get_models(api_key) @@ -52,12 +53,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 2049, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "meta/llama-3.3-70b-instruct", system_prompt: str = "You are a helpful assistant.", temperature: float = 0.7, @@ -123,9 +124,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: """ Sends a prompt to the Nvidia API and returns the response. """ @@ -151,7 +153,7 @@ def ask( } def for_stream(): - streaming_text = "" + streaming_text = "" try: response = self.session.post( self.url, @@ -175,7 +177,7 @@ def for_stream(): for content_chunk in processed_stream: if isinstance(content_chunk, bytes): content_chunk = content_chunk.decode('utf-8', errors='ignore') - + if raw: yield content_chunk else: @@ -234,7 +236,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, raw: bool = False, conversationally: bool = False, ) -> Union[str, Generator[str, None, None]]: @@ -262,8 +264,9 @@ def for_non_stream_chat(): return for_stream_chat() if stream else for_non_stream_chat() - def get_message(self, response: dict) -> str: - assert isinstance(response, dict), "Response should be of dict data-type only" + def get_message(self, response: Response) -> str: + if not isinstance(response, dict): + return str(response) return response.get("text", "") if __name__ == "__main__": diff --git a/webscout/Provider/OPENAI/DeepAI.py b/webscout/Provider/OPENAI/DeepAI.py index 54179e21..f2c27a23 100644 --- a/webscout/Provider/OPENAI/DeepAI.py +++ b/webscout/Provider/OPENAI/DeepAI.py @@ -1,18 +1,23 @@ -from curl_cffi.requests import Session, RequestsError -from typing import List, Dict, Optional, Union, Generator, Any - -# Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions -from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, format_prompt, count_tokens -) - # Standard library imports import json import time import uuid +from typing import Any, Dict, Generator, List, Optional, Union + +from curl_cffi.requests import RequestsError, Session + +# Import base classes and utility structures +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider +from webscout.Provider.OPENAI.utils import ( + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, +) # Attempt to import LitAgent, fallback if not available try: @@ -308,7 +313,7 @@ def __init__( self.session.cookies.update(self.cookies) - def refresh_identity(self, browser: str = None, impersonate: str = "chrome120"): + def refresh_identity(self, browser: Optional[str] = None, impersonate: str = "chrome120"): """Refreshes the browser identity fingerprint and curl_cffi session.""" browser = browser or self.fingerprint.get("browser_type", "chrome") self.fingerprint = LitAgent().generate_fingerprint(browser) @@ -333,7 +338,7 @@ def refresh_identity(self, browser: str = None, impersonate: str = "chrome120"): return self.fingerprint @classmethod - def get_models(cls, api_key: str = None): + def get_models(cls, api_key: Optional[str] = None): """Fetch available models from DeepAI API. Args: @@ -375,16 +380,8 @@ def get_models(cls, api_key: str = None): try: # Use a temporary session for this class method from curl_cffi.requests import Session - temp_session = Session() - - headers = { - "Content-Type": "application/x-www-form-urlencoded", - "api-key": api_key, - "Accept": "*/*", - "Accept-Language": "en-US,en;q=0.9", - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", - "DNT": "1", - } + Session() + # Note: DeepAI doesn't have a standard models endpoint, so we'll use a default list # If DeepAI has a models endpoint, you would call it here @@ -474,4 +471,8 @@ def list(inner_self): messages=[{"role": "user", "content": "Hello!"}], stream=False ) - print(response.choices[0].message.content) \ No newline at end of file + if isinstance(response, ChatCompletion): + print(response.choices[0].message.content) + else: + for chunk in response: + print(chunk.choices[0].delta.content, end="") diff --git a/webscout/Provider/OPENAI/K2Think.py b/webscout/Provider/OPENAI/K2Think.py index 864fbc47..7078a495 100644 --- a/webscout/Provider/OPENAI/K2Think.py +++ b/webscout/Provider/OPENAI/K2Think.py @@ -1,21 +1,24 @@ +import json +import re import time import uuid -import requests -import re -import json -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union + +# Import LitAgent +from webscout.litagent import LitAgent # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, ) -# Import LitAgent -from webscout.litagent import LitAgent - -from litprinter import ic class Completions(BaseCompletions): def __init__(self, client: 'K2Think'): @@ -38,7 +41,7 @@ def create( Mimics openai.chat.completions.create """ # Prepare the payload for K2Think API - payload = { + payload: Dict[str, Any] = { "stream": stream, "model": model, "messages": messages, @@ -98,22 +101,22 @@ def _create_stream( extract_regexes = [ r'([\s\S]*?)<\/answer>', ] - + content = "" for regex in extract_regexes: match = re.search(regex, decoded_line) if match: content = match.group(1) break - + if content: # Format the content content = self._client.format_text(content) - + # Skip if we've already seen this exact content if content in seen_content: continue - + seen_content.add(content) # Update token counts using count_tokens @@ -212,14 +215,14 @@ def _create_non_stream( # Collect the full response full_text = "" seen_content_parts = set() # Track seen content parts to avoid duplicates - + for line in response.iter_lines(decode_unicode=True): if line: # Extract content using regex patterns extract_regexes = [ r'([\s\S]*?)<\/answer>', ] - + for regex in extract_regexes: match = re.search(regex, line) if match: @@ -283,7 +286,7 @@ def __init__(self): self.available_models = [ "MBZUAI-IFM/K2-Think", ] - + def list(self): """Return list of available models""" return [ @@ -423,6 +426,9 @@ def K2ThinkClient(**kwargs): stream=True ) - for chunk in response: - if chunk.choices[0].delta.content: - print(chunk.choices[0].delta.content, end='', flush=True) \ No newline at end of file + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes, ChatCompletion)): + for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end='', flush=True) + else: + print(response) diff --git a/webscout/Provider/OPENAI/PI.py b/webscout/Provider/OPENAI/PI.py index 913f0339..b775f93b 100644 --- a/webscout/Provider/OPENAI/PI.py +++ b/webscout/Provider/OPENAI/PI.py @@ -1,24 +1,29 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError import json -import time -import uuid import re import threading -from typing import List, Dict, Optional, Union, Generator, Any +import time +import uuid +from typing import Any, Dict, Generator, List, Optional, Union from uuid import uuid4 -# Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions -from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage -) +from curl_cffi import CurlError +from curl_cffi.requests import Session + +from webscout import exceptions # Attempt to import LitAgent, fallback if not available from webscout.litagent import LitAgent -from webscout import exceptions +# Import base classes and utility structures +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider +from webscout.Provider.OPENAI.utils import ( + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, +) # --- PI.ai Client --- @@ -51,9 +56,9 @@ def create( raise ValueError(f"Voice '{voice_name}' not available. Choose from: {list(self._client.AVAILABLE_VOICES.keys())}") # Use format_prompt from utils.py to convert OpenAI messages format to Pi.ai prompt - from webscout.Provider.OPENAI.utils import format_prompt, count_tokens + from webscout.Provider.OPENAI.utils import count_tokens, format_prompt prompt = format_prompt(messages, do_continue=True, add_special_tokens=True) - + # Ensure conversation is started if not self._client.conversation_id: self._client.start_conversation() @@ -66,12 +71,12 @@ def create( if stream: return self._create_stream( - request_id, created_time, model, prompt, + request_id, created_time, model, prompt, timeout, proxies, voice, voice_name, output_file, prompt_tokens ) else: return self._create_non_stream( - request_id, created_time, model, prompt, + request_id, created_time, model, prompt, timeout, proxies, voice, voice_name, output_file, prompt_tokens ) @@ -81,9 +86,9 @@ def _create_stream( voice: bool = False, voice_name: str = "voice3", output_file: str = "PiAI.mp3", prompt_tokens: Optional[int] = None ) -> Generator[ChatCompletionChunk, None, None]: - + from webscout.Provider.OPENAI.utils import count_tokens - + data = { 'text': prompt, 'conversation': self._client.conversation_id @@ -116,7 +121,6 @@ def _create_stream( # Track token usage across chunks # prompt_tokens = len(prompt.split()) if prompt else 0 completion_tokens = 0 - total_tokens = prompt_tokens sids = [] streaming_text = "" @@ -127,19 +131,18 @@ def _create_stream( if line_bytes: line = line_bytes.decode('utf-8') full_raw_data_for_sids += line + "\n" - + if line.startswith("data: "): json_line_str = line[6:] try: chunk_data = json.loads(json_line_str) content = chunk_data.get('text', '') - + if content: # Assume content is incremental (new text since last chunk) new_content = content streaming_text += new_content completion_tokens += count_tokens(new_content) if new_content else 0 - total_tokens = prompt_tokens + completion_tokens # Create OpenAI-compatible chunk delta = ChoiceDelta( @@ -202,9 +205,9 @@ def _create_non_stream( voice: bool = False, voice_name: str = "voice3", output_file: str = "PiAI.mp3", prompt_tokens: Optional[int] = None ) -> ChatCompletion: - + from webscout.Provider.OPENAI.utils import count_tokens - + # Collect streaming response into a single response full_content = "" # prompt_tokens = len(prompt.split()) if prompt else 0 # replaced @@ -214,7 +217,7 @@ def _create_non_stream( prompt_tokens = count_tokens(prompt) for chunk in self._create_stream( - request_id, created_time, model, prompt, + request_id, created_time, model, prompt, timeout, proxies, voice, voice_name, output_file, prompt_tokens ): if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.content: @@ -260,7 +263,7 @@ def __init__(self, client: 'PiAI'): class PiAI(OpenAICompatibleProvider): """ PiAI provider following OpenAI-compatible interface. - + Supports Pi.ai specific features like voice generation and conversation management. """ required_auth = False @@ -277,7 +280,7 @@ class PiAI(OpenAICompatibleProvider): } def __init__( - self, + self, api_key: Optional[str] = None, timeout: int = 30, proxies: Optional[Dict[str, str]] = None, @@ -285,7 +288,7 @@ def __init__( ): """ Initialize PI.ai provider. - + Args: api_key: Not used for Pi.ai but kept for compatibility timeout: Request timeout in seconds @@ -295,11 +298,11 @@ def __init__( super().__init__(proxies=proxies) self.timeout = timeout self.conversation_id = None - + # Setup URLs self.primary_url = 'https://pi.ai/api/chat' self.fallback_url = 'https://pi.ai/api/v2/chat' - + # Setup headers self.headers = { 'Accept': 'text/event-stream', @@ -315,27 +318,27 @@ def __init__( 'User-Agent': LitAgent().random(), 'X-Api-Version': '3' } - + # Setup cookies self.cookies = { '__cf_bm': uuid4().hex } - + # Replace the base session with curl_cffi Session self.session = Session() - + # Configure session self.session.headers.update(self.headers) if proxies: self.session.proxies = proxies - + # Set cookies on the session for name, value in self.cookies.items(): self.session.cookies.set(name, value) - + # Initialize chat interface self.chat = Chat(self) - + # Start conversation self.start_conversation() @@ -401,7 +404,7 @@ def list(inner_self): if __name__ == "__main__": # Test the OpenAI-compatible interface client = PiAI() - + # Test streaming print("Testing streaming response:") response = client.chat.completions.create( @@ -411,12 +414,15 @@ def list(inner_self): ], stream=True ) - - for chunk in response: - if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.content: - print(chunk.choices[0].delta.content, end="", flush=True) + + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes, ChatCompletion)): + for chunk in response: + if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="", flush=True) + else: + print(response) print() - + # Test non-streaming print("\nTesting non-streaming response:") response = client.chat.completions.create( @@ -426,6 +432,9 @@ def list(inner_self): ], stream=False ) - - print(response.choices[0].message.content) - print(f"Usage: {response.usage}") \ No newline at end of file + + if isinstance(response, ChatCompletion): + print(response.choices[0].message.content) + print(f"Usage: {response.usage}") + else: + print(response) diff --git a/webscout/Provider/OPENAI/TogetherAI.py b/webscout/Provider/OPENAI/TogetherAI.py index 0770fa79..2d5d59f0 100644 --- a/webscout/Provider/OPENAI/TogetherAI.py +++ b/webscout/Provider/OPENAI/TogetherAI.py @@ -42,7 +42,7 @@ def create( """ # Get API key if not already set if not self._client.headers.get("Authorization"): - # If no API key is set, we can't proceed. + # If no API key is set, we can't proceed. # The user should have provided it in __init__. pass @@ -224,23 +224,23 @@ def get_models(cls, api_key: str = None): "Authorization": f"Bearer {api_key}", "Content-Type": "application/json" } - + response = requests.get( "https://api.together.xyz/v1/models", headers=headers, timeout=30 ) - + if response.status_code != 200: return cls.AVAILABLE_MODELS - + data = response.json() # Together API returns a list of model objects if isinstance(data, list): return [model["id"] for model in data if isinstance(model, dict) and "id" in model] - + return cls.AVAILABLE_MODELS - + except Exception: return cls.AVAILABLE_MODELS @@ -261,13 +261,13 @@ def __init__(self, api_key: str = None, browser: str = "chrome", proxies: Option # Initialize LitAgent for consistent fingerprints across requests self._agent = LitAgent() self.headers = self._generate_consistent_fingerprint(browser=browser) - + if api_key: self.headers["Authorization"] = f"Bearer {api_key}" - + self.session.headers.update(self.headers) self.chat = Chat(self) - + # Try to update models if API key is provided if api_key: try: diff --git a/webscout/Provider/OPENAI/TwoAI.py b/webscout/Provider/OPENAI/TwoAI.py index f3a8e806..ed09d98d 100644 --- a/webscout/Provider/OPENAI/TwoAI.py +++ b/webscout/Provider/OPENAI/TwoAI.py @@ -1,15 +1,17 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError import json import time import uuid -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union # Import base classes and utilities from OPENAI provider stack -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, ) @@ -246,4 +248,4 @@ def list(inner_self): stream=True ) for chunk in resp: - print(chunk, end="") \ No newline at end of file + print(chunk, end="") diff --git a/webscout/Provider/OPENAI/__init__.py b/webscout/Provider/OPENAI/__init__.py index b72dea98..ff239b16 100644 --- a/webscout/Provider/OPENAI/__init__.py +++ b/webscout/Provider/OPENAI/__init__.py @@ -2,77 +2,75 @@ # Static imports for all OPENAI provider modules # Base classes and utilities +from webscout.Provider.OPENAI.ai4chat import AI4Chat +from webscout.Provider.OPENAI.akashgpt import AkashGPT +from webscout.Provider.OPENAI.algion import Algion +from webscout.Provider.OPENAI.ayle import Ayle from webscout.Provider.OPENAI.base import ( - OpenAICompatibleProvider, BaseChat, BaseCompletions, + FunctionDefinition, + FunctionParameters, + OpenAICompatibleProvider, Tool, ToolDefinition, - FunctionParameters, - FunctionDefinition, ) - -from webscout.Provider.OPENAI.utils import ( - ChatCompletion, - ChatCompletionChunk, - Choice, - ChoiceDelta, - ChatCompletionMessage, - CompletionUsage, - ToolCall, - ToolFunction, - FunctionCall, - ToolCallType, - ModelData, - ModelList, - format_prompt, - get_system_prompt, - get_last_user_message, - count_tokens, -) - -# Provider implementations -from webscout.Provider.OPENAI.DeepAI import DeepAI -from webscout.Provider.OPENAI.hadadxyz import HadadXYZ -from webscout.Provider.OPENAI.K2Think import K2Think -from webscout.Provider.OPENAI.PI import PiAI -from webscout.Provider.OPENAI.TogetherAI import TogetherAI -from webscout.Provider.OPENAI.TwoAI import TwoAI -from webscout.Provider.OPENAI.ai4chat import AI4Chat -from webscout.Provider.OPENAI.akashgpt import AkashGPT -from webscout.Provider.OPENAI.algion import Algion from webscout.Provider.OPENAI.cerebras import Cerebras from webscout.Provider.OPENAI.chatgpt import ChatGPT, ChatGPTReversed from webscout.Provider.OPENAI.chatsandbox import ChatSandbox + +# Provider implementations +from webscout.Provider.OPENAI.DeepAI import DeepAI from webscout.Provider.OPENAI.deepinfra import DeepInfra from webscout.Provider.OPENAI.e2b import E2B from webscout.Provider.OPENAI.elmo import Elmo from webscout.Provider.OPENAI.exaai import ExaAI from webscout.Provider.OPENAI.freeassist import FreeAssist -from webscout.Provider.OPENAI.ayle import Ayle -from webscout.Provider.OPENAI.huggingface import HuggingFace +from webscout.Provider.OPENAI.gradient import Gradient from webscout.Provider.OPENAI.groq import Groq +from webscout.Provider.OPENAI.hadadxyz import HadadXYZ from webscout.Provider.OPENAI.heckai import HeckAI +from webscout.Provider.OPENAI.huggingface import HuggingFace from webscout.Provider.OPENAI.ibm import IBM +from webscout.Provider.OPENAI.K2Think import K2Think from webscout.Provider.OPENAI.llmchat import LLMChat from webscout.Provider.OPENAI.llmchatco import LLMChatCo +from webscout.Provider.OPENAI.meta import Meta from webscout.Provider.OPENAI.netwrck import Netwrck from webscout.Provider.OPENAI.nvidia import Nvidia from webscout.Provider.OPENAI.oivscode import oivscode +from webscout.Provider.OPENAI.PI import PiAI +from webscout.Provider.OPENAI.sambanova import Sambanova from webscout.Provider.OPENAI.sonus import SonusAI from webscout.Provider.OPENAI.textpollinations import TextPollinations +from webscout.Provider.OPENAI.TogetherAI import TogetherAI from webscout.Provider.OPENAI.toolbaz import Toolbaz +from webscout.Provider.OPENAI.TwoAI import TwoAI from webscout.Provider.OPENAI.typefully import TypefullyAI +from webscout.Provider.OPENAI.typliai import TypliAI +from webscout.Provider.OPENAI.utils import ( + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + FunctionCall, + ModelData, + ModelList, + ToolCall, + ToolCallType, + ToolFunction, + count_tokens, + format_prompt, + get_last_user_message, + get_system_prompt, +) from webscout.Provider.OPENAI.venice import Venice from webscout.Provider.OPENAI.wisecat import WiseCat from webscout.Provider.OPENAI.writecream import Writecream from webscout.Provider.OPENAI.x0gpt import X0GPT -from webscout.Provider.OPENAI.yep import YEPCHAT from webscout.Provider.OPENAI.zenmux import Zenmux -from webscout.Provider.OPENAI.gradient import Gradient -from webscout.Provider.OPENAI.sambanova import Sambanova -from webscout.Provider.OPENAI.meta import Meta -from webscout.Provider.OPENAI.typliai import TypliAI # List of all exported names __all__ = [ diff --git a/webscout/Provider/OPENAI/ai4chat.py b/webscout/Provider/OPENAI/ai4chat.py index 5a4d9af1..b39a56c8 100644 --- a/webscout/Provider/OPENAI/ai4chat.py +++ b/webscout/Provider/OPENAI/ai4chat.py @@ -1,14 +1,20 @@ import time -import uuid import urllib.parse -from curl_cffi.requests import Session, RequestsError -from typing import List, Dict, Optional, Union, Generator, Any +import uuid +from typing import Any, Dict, Generator, List, Optional, Union + +from curl_cffi.requests import RequestsError # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, ) # --- AI4Chat Client --- @@ -65,7 +71,7 @@ def _create_stream( full_response = self._get_ai4chat_response(conversation_prompt, country, user_id, timeout=timeout, proxies=proxies) # Track token usage - prompt_tokens = count_tokens(conversation_prompt) + count_tokens(conversation_prompt) completion_tokens = 0 # Stream fixed-size character chunks (e.g., 48 chars) @@ -194,7 +200,7 @@ def _get_ai4chat_response(self, prompt: str, country: str, user_id: str, original_proxies = self._client.session.proxies if proxies is not None: self._client.session.proxies = proxies - + try: # URL encode parameters encoded_text = urllib.parse.quote(prompt) @@ -299,7 +305,7 @@ class _ModelList: def list(inner_self): return type(self).AVAILABLE_MODELS return _ModelList() - + if __name__ == "__main__": # Example usage client = AI4Chat() @@ -310,4 +316,4 @@ def list(inner_self): {"role": "user", "content": "Hello, how are you?"} ] ) - print(response.choices[0].message.content) \ No newline at end of file + print(response.choices[0].message.content) diff --git a/webscout/Provider/OPENAI/akashgpt.py b/webscout/Provider/OPENAI/akashgpt.py index 061209a0..1f0b5b1f 100644 --- a/webscout/Provider/OPENAI/akashgpt.py +++ b/webscout/Provider/OPENAI/akashgpt.py @@ -1,20 +1,24 @@ +import json +import re import time import uuid -import requests -import re -import json -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union + +# Import LitAgent for user agent generation +from webscout.litagent import LitAgent # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions, Tool +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, + format_prompt, ) -from webscout.Provider.OPENAI.utils import format_prompt, get_system_prompt - -# Import LitAgent for user agent generation -from webscout.litagent import LitAgent # AkashGPT constants AVAILABLE_MODELS = [ @@ -58,7 +62,7 @@ def create( conversation_prompt = format_prompt(messages, add_special_tokens=True, include_system=True) # Set up request parameters - api_key = kwargs.get("api_key", self._client.api_key) + kwargs.get("api_key", self._client.api_key) # Generate request ID and timestamp request_id = str(uuid.uuid4()) @@ -312,7 +316,7 @@ class AkashGPT(OpenAICompatibleProvider): print(response.choices[0].message.content) """ required_auth = True - + AVAILABLE_MODELS = AVAILABLE_MODELS def __init__( @@ -330,7 +334,7 @@ def __init__( proxies: Optional proxy configuration dict """ super().__init__(api_key=api_key, tools=tools, proxies=proxies) - + # Store the api_key for use in completions self.api_key = api_key self.timeout = 30 @@ -383,4 +387,4 @@ def list(inner_self): messages=[{"role": "user", "content": "Hello! How are you?"}] ) print(response.choices[0].message.content) - print(f"Usage: {response.usage}") \ No newline at end of file + print(f"Usage: {response.usage}") diff --git a/webscout/Provider/OPENAI/algion.py b/webscout/Provider/OPENAI/algion.py index daf96389..a1bd4dc0 100644 --- a/webscout/Provider/OPENAI/algion.py +++ b/webscout/Provider/OPENAI/algion.py @@ -1,15 +1,19 @@ import json import time import uuid -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union -from curl_cffi.requests import Session from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, ) try: @@ -207,7 +211,7 @@ class Algion(OpenAICompatibleProvider): def get_models(cls, api_key: str = None): """Fetch available models from Algion API.""" api_key = api_key or "123123" - + try: # Use a temporary curl_cffi session for this class method temp_session = Session() @@ -215,21 +219,21 @@ def get_models(cls, api_key: str = None): "Content-Type": "application/json", "Authorization": f"Bearer {api_key}", } - + response = temp_session.get( "https://api.algion.dev/v1/models", headers=headers, impersonate="chrome110" ) - + if response.status_code != 200: raise Exception(f"Failed to fetch models: HTTP {response.status_code}") - + data = response.json() if "data" in data and isinstance(data["data"], list): return [model["id"] for model in data["data"]] raise Exception("Invalid response format from API") - + except (CurlError, Exception) as e: raise Exception(f"Failed to fetch models: {str(e)}") @@ -257,7 +261,7 @@ def __init__(self, browser: str = "chrome", api_key: str = "123123"): self.headers["Authorization"] = f"Bearer {api_key}" self.session.headers.update(self.headers) self.chat = Chat(self) - + @property def models(self): class _ModelList: @@ -278,4 +282,4 @@ def list(inner_self): max_tokens=1000, stream=False ) - print(response.choices[0].message.content) \ No newline at end of file + print(response.choices[0].message.content) diff --git a/webscout/Provider/OPENAI/ayle.py b/webscout/Provider/OPENAI/ayle.py index e63a63da..6d5a3d83 100644 --- a/webscout/Provider/OPENAI/ayle.py +++ b/webscout/Provider/OPENAI/ayle.py @@ -1,20 +1,20 @@ +import json import time import uuid +from typing import Any, Dict, Generator, List, Optional, Union + import requests -import json -from typing import List, Dict, Optional, Union, Generator, Any from webscout.litagent import LitAgent from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( ChatCompletion, ChatCompletionChunk, - Choice, ChatCompletionMessage, + Choice, ChoiceDelta, CompletionUsage, - format_prompt, - count_tokens + count_tokens, ) # ANSI escape codes for formatting @@ -28,7 +28,7 @@ "endpoint": "https://ayle.chat/api/chat", "models": [ "gemini-2.5-flash", - "llama-3.3-70b-versatile", + "llama-3.3-70b-versatile", "llama-3.3-70b", "tngtech/deepseek-r1t2-chimera:free", "openai/gpt-oss-120b", @@ -64,7 +64,7 @@ def create( """ # Determine the provider based on the model provider = self._client._get_provider_from_model(model) - + # Build the appropriate payload payload = { "messages": messages, @@ -101,7 +101,7 @@ def _create_stream( for line in response.iter_lines(): if not line: continue - + try: line_str = line.decode('utf-8') if line_str.startswith('0:"'): @@ -109,18 +109,18 @@ def _create_stream( if content: streaming_text += content completion_tokens += count_tokens(content) - + # Create a delta object for this chunk delta = ChoiceDelta(content=content) choice = Choice(index=0, delta=delta, finish_reason=None) - + chunk = ChatCompletionChunk( id=request_id, choices=[choice], created=created_time, model=model, ) - + yield chunk except (json.JSONDecodeError, UnicodeDecodeError): continue @@ -128,14 +128,14 @@ def _create_stream( # Final chunk with finish_reason delta = ChoiceDelta(content=None) choice = Choice(index=0, delta=delta, finish_reason="stop") - + chunk = ChatCompletionChunk( id=request_id, choices=[choice], created=created_time, model=model, ) - + yield chunk except requests.exceptions.RequestException as e: @@ -172,26 +172,26 @@ def _create_non_stream( prompt_tokens = count_tokens(str(payload.get("messages", ""))) completion_tokens = count_tokens(full_response) total_tokens = prompt_tokens + completion_tokens - + usage = CompletionUsage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=total_tokens ) - + # Create the message object message = ChatCompletionMessage( role="assistant", content=full_response ) - + # Create the choice object choice = Choice( index=0, message=message, finish_reason="stop" ) - + # Create the completion object completion = ChatCompletion( id=request_id, @@ -200,7 +200,7 @@ def _create_non_stream( model=model, usage=usage, ) - + return completion except Exception as e: @@ -226,7 +226,7 @@ class Ayle(OpenAICompatibleProvider): required_auth = False AVAILABLE_MODELS = [ "gemini-2.5-flash", - "llama-3.3-70b-versatile", + "llama-3.3-70b-versatile", "llama-3.3-70b", "tngtech/deepseek-r1t2-chimera:free", "openai/gpt-oss-120b", @@ -253,10 +253,10 @@ def __init__( self.timeout = timeout self.temperature = temperature self.top_p = top_p - + # Initialize LitAgent for user agent generation agent = LitAgent() - + self.headers = { "accept": "*/*", "accept-language": "en-US,en;q=0.9", @@ -265,11 +265,11 @@ def __init__( "referer": "https://ayle.chat/", "user-agent": agent.random(), } - + self.session = requests.Session() self.session.headers.update(self.headers) self.session.cookies.update({"session": uuid.uuid4().hex}) - + # Initialize the chat interface self.chat = Chat(self) @@ -288,7 +288,7 @@ def _get_provider_from_model(self, model: str) -> str: for provider, config in MODEL_CONFIGS.items(): if model in config["models"]: return provider - + # If model not found, use a default model print(f"{BOLD}Warning: Model '{model}' not found, using default model 'ayle'{RESET}") return "ayle" @@ -299,12 +299,12 @@ def convert_model_name(self, model: str) -> str: """ if model in self.AVAILABLE_MODELS: return model - + # Try to find a matching model for available_model in self.AVAILABLE_MODELS: if model.lower() in available_model.lower(): return available_model - + # Default to gemini-2.5-flash if no match print(f"{BOLD}Warning: Model '{model}' not found, using default model 'gemini-2.5-flash'{RESET}") return "gemini-2.5-flash" @@ -333,7 +333,7 @@ def convert_model_name(self, model: str) -> str: ], stream=False ) - + if response and response.choices and response.choices[0].message.content: status = "✓" # Truncate response if too long diff --git a/webscout/Provider/OPENAI/base.py b/webscout/Provider/OPENAI/base.py index 17e8091f..4ea230fc 100644 --- a/webscout/Provider/OPENAI/base.py +++ b/webscout/Provider/OPENAI/base.py @@ -1,13 +1,15 @@ -from abc import ABC, abstractmethod -from typing import List, Dict, Optional, Union, Generator, Any, TypedDict, Callable import json -import requests +from abc import ABC, abstractmethod from dataclasses import dataclass +from typing import Any, Callable, Dict, Generator, List, Optional, TypedDict, Union + +import requests from litprinter import ic # Import the utils for response structures from webscout.Provider.OPENAI.utils import ChatCompletion, ChatCompletionChunk + # Define tool-related structures class ToolDefinition(TypedDict): """Definition of a tool that can be called by the AI""" @@ -32,9 +34,9 @@ class Tool: name: str description: str parameters: Dict[str, Dict[str, Any]] - required_params: List[str] = None + required_params: Optional[List[str]] = None implementation: Optional[Callable] = None - + def to_dict(self) -> ToolDefinition: """Convert to OpenAI tool definition format""" function_def = { @@ -46,21 +48,22 @@ def to_dict(self) -> ToolDefinition: "required": self.required_params or list(self.parameters.keys()) } } - + return { "type": "function", "function": function_def } - + def execute(self, arguments: Dict[str, Any]) -> Any: """Execute the tool with the given arguments""" if not self.implementation: return f"Tool '{self.name}' does not have an implementation." - + try: return self.implementation(**arguments) except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Error executing tool '{self.name}': {str(e)}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Error executing tool '{self.name}': {str(e)}") return f"Error executing tool '{self.name}': {str(e)}" class BaseCompletions(ABC): @@ -82,7 +85,7 @@ def create( ) -> Union[ChatCompletion, Generator[ChatCompletionChunk, None, None]]: """ Abstract method to create chat completions with tool support. - + Args: model: The model to use for completion messages: List of message dictionaries @@ -93,16 +96,16 @@ def create( tools: List of tool definitions available for the model to use tool_choice: Control over which tool the model should use **kwargs: Additional model-specific parameters - + Returns: Either a completion object or a generator of completion chunks if streaming """ raise NotImplementedError - + def format_tool_calls(self, tools: List[Union[Tool, Dict[str, Any]]]) -> List[Dict[str, Any]]: """Convert tools to the format expected by the provider""" formatted_tools = [] - + for tool in tools: if isinstance(tool, Tool): formatted_tools.append(tool.to_dict()) @@ -110,30 +113,31 @@ def format_tool_calls(self, tools: List[Union[Tool, Dict[str, Any]]]) -> List[Di # Assume already formatted correctly formatted_tools.append(tool) else: - ic.configureOutput(prefix='WARNING| '); ic(f"Skipping invalid tool type: {type(tool)}") - + ic.configureOutput(prefix='WARNING| ') + ic(f"Skipping invalid tool type: {type(tool)}") + return formatted_tools - + def process_tool_calls(self, tool_calls: List[Dict[str, Any]], available_tools: Dict[str, Tool]) -> List[Dict[str, Any]]: """ Process tool calls and execute the relevant tools. - + Args: tool_calls: List of tool calls from the model available_tools: Dictionary mapping tool names to their implementations - + Returns: List of results from executing the tools """ results = [] - + for call in tool_calls: try: function_call = call.get("function", {}) tool_name = function_call.get("name") arguments_str = function_call.get("arguments", "{}") - - # Parse arguments + + # Parse arguments try: if isinstance(arguments_str, str): arguments = json.loads(arguments_str) @@ -145,7 +149,7 @@ def process_tool_calls(self, tool_calls: List[Dict[str, Any]], available_tools: "result": f"Error: Could not parse arguments JSON: {arguments_str}" }) continue - + # Execute the tool if available if tool_name in available_tools: tool_result = available_tools[tool_name].execute(arguments) @@ -159,12 +163,13 @@ def process_tool_calls(self, tool_calls: List[Dict[str, Any]], available_tools: "result": f"Error: Tool '{tool_name}' not found." }) except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Error processing tool call: {str(e)}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Error processing tool call: {str(e)}") results.append({ "tool_call_id": call.get("id", "unknown"), "result": f"Error processing tool call: {str(e)}" }) - + return results @@ -201,34 +206,34 @@ def models(self): Subclasses must implement this property. """ pass - + def register_tools(self, tools: List[Tool]) -> None: """ Register tools with the provider. - + Args: tools: List of Tool objects to register """ for tool in tools: self.available_tools[tool.name] = tool - + def get_tool_by_name(self, name: str) -> Optional[Tool]: """Get a tool by name""" return self.available_tools.get(name) - + def format_tool_response(self, messages: List[Dict[str, Any]], tool_results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """ Format tool results as messages to be sent back to the provider. - + Args: messages: The original messages tool_results: Results from executing tools - + Returns: Updated message list with tool results """ updated_messages = messages.copy() - + # Find the assistant message with tool calls for i, msg in enumerate(reversed(updated_messages)): if msg.get("role") == "assistant" and "tool_calls" in msg: @@ -241,5 +246,5 @@ def format_tool_response(self, messages: List[Dict[str, Any]], tool_results: Lis } updated_messages.append(tool_message) break - - return updated_messages \ No newline at end of file + + return updated_messages diff --git a/webscout/Provider/OPENAI/cerebras.py b/webscout/Provider/OPENAI/cerebras.py index a38b5553..3b7993d0 100644 --- a/webscout/Provider/OPENAI/cerebras.py +++ b/webscout/Provider/OPENAI/cerebras.py @@ -1,15 +1,19 @@ import json import time import uuid -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union -from curl_cffi.requests import Session from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, ) try: @@ -208,7 +212,7 @@ def get_models(cls, api_key: str = None): """Fetch available models from Cerebras API.""" if not api_key: raise Exception("API key required to fetch models") - + try: # Use a temporary curl_cffi session for this class method temp_session = Session() @@ -216,21 +220,21 @@ def get_models(cls, api_key: str = None): "Content-Type": "application/json", "Authorization": f"Bearer {api_key}", } - + response = temp_session.get( "https://api.cerebras.ai/v1/models", headers=headers, impersonate="chrome120" ) - + if response.status_code != 200: raise Exception(f"Failed to fetch models: HTTP {response.status_code}") - + data = response.json() if "data" in data and isinstance(data["data"], list): return [model['id'] for model in data['data']] raise Exception("Invalid response format from API") - + except (CurlError, Exception) as e: raise Exception(f"Failed to fetch models: {str(e)}") @@ -254,7 +258,7 @@ def __init__(self, browser: str = "chrome", api_key: str = None): } self.session.headers.update(self.headers) self.chat = Chat(self) - + @property def models(self): class _ModelList: @@ -272,4 +276,4 @@ def list(inner_self): max_tokens=1000, stream=False ) - print(response.choices[0].message.content) \ No newline at end of file + print(response.choices[0].message.content) diff --git a/webscout/Provider/OPENAI/chatgpt.py b/webscout/Provider/OPENAI/chatgpt.py index 5277a26c..d510cc27 100644 --- a/webscout/Provider/OPENAI/chatgpt.py +++ b/webscout/Provider/OPENAI/chatgpt.py @@ -1,18 +1,24 @@ +import base64 +import hashlib +import json +import random import time import uuid +from datetime import datetime, timezone +from typing import Any, Dict, Generator, List, Optional, Union + import requests -import json -import random -import base64 -import hashlib -from datetime import datetime, timedelta, timezone -from typing import List, Dict, Optional, Union, Generator, Any # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions, Tool +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider, Tool from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, ) # ANSI escape codes for formatting @@ -129,24 +135,24 @@ def simulate_bypass_headers(self, accept, spoof_address=False, pre_oai_uuid=None def generate_proof_token(self, seed: str, difficulty: str, user_agent: str = None): """ Improved proof-of-work implementation based on gpt4free/g4f/Provider/openai/proofofwork.py - + Args: seed: The seed string for the challenge difficulty: The difficulty hex string user_agent: Optional user agent string - + Returns: The proof token starting with 'gAAAAAB' """ if user_agent is None: user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36" - + screen = random.choice([3008, 4010, 6000]) * random.choice([1, 2, 4]) - + # Get current UTC time now_utc = datetime.now(timezone.utc) parse_time = now_utc.strftime('%a, %d %b %Y %H:%M:%S GMT') - + proof_token = [ screen, parse_time, None, 0, user_agent, @@ -179,14 +185,14 @@ def solve_sentinel_challenge(self, seed, difficulty): def generate_fake_sentinel_token(self): """Generate a fake sentinel token for initial authentication.""" prefix = "gAAAAAC" - + # More realistic screen sizes screen = random.choice([3008, 4010, 6000]) * random.choice([1, 2, 4]) - + # Get current UTC time now_utc = datetime.now(timezone.utc) parse_time = now_utc.strftime('%a, %d %b %Y %H:%M:%S GMT') - + config = [ screen, parse_time, @@ -219,7 +225,7 @@ def parse_response(self, input_text): json_data["message"].get("status") == "finished_successfully" and json_data["message"].get("metadata", {}).get("is_complete")): return json_data["message"]["content"]["parts"][0] - except: + except Exception: pass return input_text # Return raw text if parsing fails or no complete message found @@ -553,7 +559,6 @@ def _create_streaming( # Track conversation state full_content = "" - finish_reason = None prompt_tokens = count_tokens(str(messages)) completion_tokens = 0 total_tokens = prompt_tokens @@ -563,7 +568,7 @@ def _create_streaming( if line: if line.startswith("data: "): data_str = line[6:] # Remove "data: " prefix - + # Handle [DONE] message if data_str.strip() == "[DONE]": # Final chunk with finish_reason @@ -582,26 +587,26 @@ def _create_streaming( } yield chunk break - + try: data = json.loads(data_str) - + # Handle different types of messages if data.get("message"): message = data["message"] - + # Handle assistant responses if message.get("author", {}).get("role") == "assistant": content_parts = message.get("content", {}).get("parts", []) if content_parts: new_content = content_parts[0] - + # Get the delta (new content since last chunk) delta_content = new_content[len(full_content):] if new_content.startswith(full_content) else new_content full_content = new_content completion_tokens = count_tokens(full_content) total_tokens = prompt_tokens + completion_tokens - + # Only yield chunk if there's new content if delta_content: delta = ChoiceDelta(content=delta_content, role="assistant") @@ -618,15 +623,15 @@ def _create_streaming( "total_tokens": total_tokens } yield chunk - + # Handle finish status if message.get("status") == "finished_successfully": - finish_reason = "stop" - + pass + elif data.get("type") == "message_stream_complete": # Stream is complete - finish_reason = "stop" - + pass + except json.JSONDecodeError: # Skip invalid JSON lines continue @@ -729,20 +734,20 @@ def _create_non_streaming( if line: if line.startswith("data: "): data_str = line[6:] # Remove "data: " prefix - + # Handle [DONE] message if data_str.strip() == "[DONE]": break - + try: data = json.loads(data_str) - + # Handle assistant responses if data.get("message") and data["message"].get("author", {}).get("role") == "assistant": content_parts = data["message"].get("content", {}).get("parts", []) if content_parts: full_response = content_parts[0] - + except json.JSONDecodeError: # Skip invalid JSON lines continue @@ -843,4 +848,4 @@ def list(inner_self): messages=[{"role": "user", "content": "How many r in strawberry"}] ) print(response.choices[0].message.content) - print() \ No newline at end of file + print() diff --git a/webscout/Provider/OPENAI/chatsandbox.py b/webscout/Provider/OPENAI/chatsandbox.py index 42feee8d..fb80428b 100644 --- a/webscout/Provider/OPENAI/chatsandbox.py +++ b/webscout/Provider/OPENAI/chatsandbox.py @@ -1,22 +1,24 @@ -from typing import List, Dict, Optional, Union, Generator, Any -import time import json +import time +from typing import Any, Dict, Generator, List, Optional, Union + +from curl_cffi.const import CurlHttpVersion +from curl_cffi.requests import Session + +from webscout import exceptions +from webscout.AIutel import sanitize_stream from webscout.litagent import LitAgent -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( ChatCompletion, ChatCompletionChunk, - Choice, ChatCompletionMessage, + Choice, ChoiceDelta, CompletionUsage, + count_tokens, format_prompt, - count_tokens ) -from curl_cffi.requests import Session -from curl_cffi.const import CurlHttpVersion -from webscout.AIutel import sanitize_stream -from webscout import exceptions # ANSI escape codes for formatting BOLD = "\033[1m" @@ -92,7 +94,7 @@ def create( session = Session() session.headers.update(headers) session.proxies = proxies if proxies is not None else {} - + def for_stream(): try: response = session.post( @@ -107,7 +109,7 @@ def for_stream(): raise exceptions.FailedToGenerateResponseError( f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}" ) - + streaming_text = "" # Use sanitize_stream with the custom extractor processed_stream = sanitize_stream( @@ -173,16 +175,16 @@ def __init__(self, client: 'ChatSandbox'): class ChatSandbox(OpenAICompatibleProvider): AVAILABLE_MODELS = [ - "openai", - "openai-gpt-4o", - "openai-o1-mini", - "deepseek", - "deepseek-r1", + "openai", + "openai-gpt-4o", + "openai-o1-mini", + "deepseek", + "deepseek-r1", "deepseek-r1-full", - "gemini", + "gemini", "gemini-thinking", - "mistral", - "mistral-large", + "mistral", + "mistral-large", "gemma-3", "llama" ] diff --git a/webscout/Provider/OPENAI/deepinfra.py b/webscout/Provider/OPENAI/deepinfra.py index 0147ab66..66089d06 100644 --- a/webscout/Provider/OPENAI/deepinfra.py +++ b/webscout/Provider/OPENAI/deepinfra.py @@ -1,14 +1,18 @@ -import requests import json import time import uuid -import collections -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union + +import requests -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, ) try: @@ -16,8 +20,9 @@ except ImportError: pass + class Completions(BaseCompletions): - def __init__(self, client: 'DeepInfra'): + def __init__(self, client: "DeepInfra"): self._client = client def create( @@ -31,7 +36,7 @@ def create( top_p: Optional[float] = None, timeout: Optional[int] = None, proxies: Optional[Dict[str, str]] = None, - **kwargs: Any + **kwargs: Any, ) -> Union[ChatCompletion, Generator[ChatCompletionChunk, None, None]]: payload = { "model": model, @@ -49,11 +54,18 @@ def create( if stream: return self._create_stream(request_id, created_time, model, payload, timeout, proxies) else: - return self._create_non_stream(request_id, created_time, model, payload, timeout, proxies) + return self._create_non_stream( + request_id, created_time, model, payload, timeout, proxies + ) def _create_stream( - self, request_id: str, created_time: int, model: str, payload: Dict[str, Any], - timeout: Optional[int] = None, proxies: Optional[Dict[str, str]] = None + self, + request_id: str, + created_time: int, + model: str, + payload: Dict[str, Any], + timeout: Optional[int] = None, + proxies: Optional[Dict[str, str]] = None, ) -> Generator[ChatCompletionChunk, None, None]: try: response = self._client.session.post( @@ -62,7 +74,7 @@ def _create_stream( json=payload, stream=True, timeout=timeout or self._client.timeout, - proxies=proxies + proxies=proxies, ) response.raise_for_status() prompt_tokens = 0 @@ -76,43 +88,45 @@ def _create_stream( break try: data = json.loads(json_str) - choices = data.get('choices') + choices = data.get("choices") if not choices and choices is not None: continue choice_data = choices[0] if choices else {} - delta_data = choice_data.get('delta', {}) - finish_reason = choice_data.get('finish_reason') - usage_data = data.get('usage', {}) + delta_data = choice_data.get("delta", {}) + finish_reason = choice_data.get("finish_reason") + usage_data = data.get("usage", {}) if usage_data: - prompt_tokens = usage_data.get('prompt_tokens', prompt_tokens) - completion_tokens = usage_data.get('completion_tokens', completion_tokens) - total_tokens = usage_data.get('total_tokens', total_tokens) - if delta_data.get('content'): + prompt_tokens = usage_data.get("prompt_tokens", prompt_tokens) + completion_tokens = usage_data.get( + "completion_tokens", completion_tokens + ) + total_tokens = usage_data.get("total_tokens", total_tokens) + if delta_data.get("content"): completion_tokens += 1 total_tokens = prompt_tokens + completion_tokens delta = ChoiceDelta( - content=delta_data.get('content'), - role=delta_data.get('role'), - tool_calls=delta_data.get('tool_calls') + content=delta_data.get("content"), + role=delta_data.get("role"), + tool_calls=delta_data.get("tool_calls"), ) choice = Choice( - index=choice_data.get('index', 0), + index=choice_data.get("index", 0), delta=delta, finish_reason=finish_reason, - logprobs=choice_data.get('logprobs') + logprobs=choice_data.get("logprobs"), ) chunk = ChatCompletionChunk( id=request_id, choices=[choice], created=created_time, model=model, - system_fingerprint=data.get('system_fingerprint') + system_fingerprint=data.get("system_fingerprint"), ) chunk.usage = { "prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens, - "estimated_cost": None + "estimated_cost": None, } yield chunk except json.JSONDecodeError: @@ -125,13 +139,13 @@ def _create_stream( choices=[choice], created=created_time, model=model, - system_fingerprint=None + system_fingerprint=None, ) chunk.usage = { "prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens, - "estimated_cost": None + "estimated_cost": None, } yield chunk except Exception as e: @@ -139,8 +153,13 @@ def _create_stream( raise IOError(f"DeepInfra request failed: {e}") from e def _create_non_stream( - self, request_id: str, created_time: int, model: str, payload: Dict[str, Any], - timeout: Optional[int] = None, proxies: Optional[Dict[str, str]] = None + self, + request_id: str, + created_time: int, + model: str, + payload: Dict[str, Any], + timeout: Optional[int] = None, + proxies: Optional[Dict[str, str]] = None, ) -> ChatCompletion: try: response = self._client.session.post( @@ -148,43 +167,42 @@ def _create_non_stream( headers=self._client.headers, json=payload, timeout=timeout or self._client.timeout, - proxies=proxies + proxies=proxies, ) response.raise_for_status() data = response.json() - choices_data = data.get('choices', []) - usage_data = data.get('usage', {}) + choices_data = data.get("choices", []) + usage_data = data.get("usage", {}) choices = [] for choice_d in choices_data: - message_d = choice_d.get('message') - if not message_d and 'delta' in choice_d: - delta = choice_d['delta'] + message_d = choice_d.get("message") + if not message_d and "delta" in choice_d: + delta = choice_d["delta"] message_d = { - 'role': delta.get('role', 'assistant'), - 'content': delta.get('content', '') + "role": delta.get("role", "assistant"), + "content": delta.get("content", ""), } if not message_d: - message_d = {'role': 'assistant', 'content': ''} + message_d = {"role": "assistant", "content": ""} message = ChatCompletionMessage( - role=message_d.get('role', 'assistant'), - content=message_d.get('content', '') + role=message_d.get("role", "assistant"), content=message_d.get("content", "") ) choice = Choice( - index=choice_d.get('index', 0), + index=choice_d.get("index", 0), message=message, - finish_reason=choice_d.get('finish_reason', 'stop') + finish_reason=choice_d.get("finish_reason", "stop"), ) choices.append(choice) usage = CompletionUsage( - prompt_tokens=usage_data.get('prompt_tokens', 0), - completion_tokens=usage_data.get('completion_tokens', 0), - total_tokens=usage_data.get('total_tokens', 0) + prompt_tokens=usage_data.get("prompt_tokens", 0), + completion_tokens=usage_data.get("completion_tokens", 0), + total_tokens=usage_data.get("total_tokens", 0), ) completion = ChatCompletion( id=request_id, choices=choices, created=created_time, - model=data.get('model', model), + model=data.get("model", model), usage=usage, ) return completion @@ -192,10 +210,12 @@ def _create_non_stream( print(f"Error during DeepInfra non-stream request: {e}") raise IOError(f"DeepInfra request failed: {e}") from e + class Chat(BaseChat): - def __init__(self, client: 'DeepInfra'): + def __init__(self, client: "DeepInfra"): self.completions = Completions(client) + class DeepInfra(OpenAICompatibleProvider): required_auth = False AVAILABLE_MODELS = [ @@ -279,7 +299,60 @@ class DeepInfra(OpenAICompatibleProvider): "allenai/olmOCR-7B-0725-FP8", ] + @classmethod + def get_models(cls, api_key: Optional[str] = None): + """Fetch available models from DeepInfra API. + + Args: + api_key (str, optional): DeepInfra API key. If not provided, returns default models. + + Returns: + list: List of available model IDs + """ + if not api_key: + return cls.AVAILABLE_MODELS + + try: + # Use a temporary requests session for this class method + temp_session = requests.Session() + headers = { + "Content-Type": "application/json", + } + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + response = temp_session.get( + "https://api.deepinfra.com/v1/models", + headers=headers, + ) + + if response.status_code != 200: + return cls.AVAILABLE_MODELS + + data = response.json() + if "data" in data and isinstance(data["data"], list): + return [model["id"] for model in data["data"]] + return cls.AVAILABLE_MODELS + + except Exception: + # Fallback to default models list if fetching fails + return cls.AVAILABLE_MODELS + + @classmethod + def update_available_models(cls, api_key=None): + """Update the available models list from DeepInfra API""" + try: + models = cls.get_models(api_key) + if models and len(models) > 0: + cls.AVAILABLE_MODELS = models + except Exception: + # Fallback to default models list if fetching fails + pass + def __init__(self, browser: str = "chrome", api_key: str = None): + # Update available models from API + self.update_available_models(api_key) + self.timeout = None self.base_url = "https://api.deepinfra.com/v1/openai/chat/completions" self.session = requests.Session() @@ -299,7 +372,8 @@ def __init__(self, browser: str = "chrome", api_key: str = None): "Sec-Fetch-Mode": "cors", "Sec-Fetch-Site": "same-site", "X-Deepinfra-Source": "web-embed", - "Sec-CH-UA": fingerprint["sec_ch_ua"] or '"Not)A;Brand";v="99", "Microsoft Edge";v="127", "Chromium";v="127"', + "Sec-CH-UA": fingerprint["sec_ch_ua"] + or '"Not)A;Brand";v="99", "Microsoft Edge";v="127", "Chromium";v="127"', "Sec-CH-UA-Mobile": "?0", "Sec-CH-UA-Platform": f'"{fingerprint["platform"]}"', "User-Agent": fingerprint["user_agent"], @@ -308,19 +382,22 @@ def __init__(self, browser: str = "chrome", api_key: str = None): self.headers["Authorization"] = f"Bearer {api_key}" self.session.headers.update(self.headers) self.chat = Chat(self) + @property def models(self): class _ModelList: def list(inner_self): - return type(self).AVAILABLE_MODELS + return DeepInfra.AVAILABLE_MODELS + return _ModelList() + if __name__ == "__main__": client = DeepInfra() response = client.chat.completions.create( model="deepseek-ai/DeepSeek-R1-0528", messages=[{"role": "user", "content": "Hello, how are you?"}], max_tokens=10000, - stream=False + stream=False, ) - print(response.choices[0].message.content) \ No newline at end of file + print(response.choices[0].message.content) diff --git a/webscout/Provider/OPENAI/e2b.py b/webscout/Provider/OPENAI/e2b.py index 38364245..fa258561 100644 --- a/webscout/Provider/OPENAI/e2b.py +++ b/webscout/Provider/OPENAI/e2b.py @@ -1627,7 +1627,6 @@ def create( def _send_request(self, request_body: dict, model_config: dict, timeout: Optional[int] = None, proxies: Optional[Dict[str, str]] = None, retries: int = 3) -> str: """Enhanced request method with IP rotation, session rotation, and advanced rate limit bypass.""" url = model_config["apiUrl"] - target_origin = "https://fragments.e2b.dev" # Use client proxies if none provided if proxies is None: @@ -2171,13 +2170,16 @@ def _build_request_body(self, model_config: dict, messages: list, system_prompt: def _merge_user_messages(self, messages: list) -> list: """Merges consecutive user messages""" - if not messages: return [] + if not messages: + return [] merged = [] current_message = messages[0] for next_message in messages[1:]: - if not isinstance(next_message, dict) or "role" not in next_message: continue + if not isinstance(next_message, dict) or "role" not in next_message: + continue if not isinstance(current_message, dict) or "role" not in current_message: - current_message = next_message; continue + current_message = next_message + continue if current_message["role"] == "user" and next_message["role"] == "user": if (isinstance(current_message.get("content"), list) and current_message["content"] and isinstance(current_message["content"][0], dict) and current_message["content"][0].get("type") == "text" and @@ -2185,23 +2187,32 @@ def _merge_user_messages(self, messages: list) -> list: isinstance(next_message["content"][0], dict) and next_message["content"][0].get("type") == "text"): current_message["content"][0]["text"] += "\n" + next_message["content"][0]["text"] else: - merged.append(current_message); current_message = next_message + merged.append(current_message) + current_message = next_message else: - merged.append(current_message); current_message = next_message - if current_message not in merged: merged.append(current_message) + merged.append(current_message) + current_message = next_message + if current_message not in merged: + merged.append(current_message) return merged def _transform_content(self, messages: list) -> list: """Transforms message format and merges consecutive user messages""" transformed = [] for msg in messages: - if not isinstance(msg, dict): continue + if not isinstance(msg, dict): + continue role, content = msg.get("role"), msg.get("content") - if role is None or content is None: continue - if isinstance(content, list): transformed.append(msg); continue + if role is None or content is None: + continue + if isinstance(content, list): + transformed.append(msg) + continue if not isinstance(content, str): - try: content = str(content) - except Exception: continue + try: + content = str(content) + except Exception: + continue base_content = {"type": "text", "text": content} # System messages are handled separately now, no need for role-playing prompt here. diff --git a/webscout/Provider/OPENAI/elmo.py b/webscout/Provider/OPENAI/elmo.py index eb6a7859..c7e1031d 100644 --- a/webscout/Provider/OPENAI/elmo.py +++ b/webscout/Provider/OPENAI/elmo.py @@ -1,18 +1,20 @@ -import json +import re import time import uuid -import re -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union from curl_cffi.requests import Session -from curl_cffi import CurlError -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.AIutel import sanitize_stream +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, ) -from webscout.AIutel import sanitize_stream try: from webscout.litagent import LitAgent @@ -105,7 +107,7 @@ def _create_stream( impersonate="chrome110" ) response.raise_for_status() - + # Use sanitize_stream to process the response processed_stream = sanitize_stream( data=response.iter_content(chunk_size=None), # Pass byte iterator @@ -119,7 +121,7 @@ def _create_stream( prompt_tokens = 0 completion_tokens = 0 total_tokens = 0 - + for content_chunk in processed_stream: if content_chunk and isinstance(content_chunk, str): completion_tokens += len(content_chunk.split()) @@ -149,7 +151,7 @@ def _create_stream( "estimated_cost": None } yield chunk - + # Final chunk delta = ChoiceDelta(content=None, role=None, tool_calls=None) choice = Choice(index=0, delta=delta, finish_reason="stop", logprobs=None) @@ -185,7 +187,7 @@ def _create_non_stream( proxies=proxies ) response.raise_for_status() - + # Use sanitize_stream to process the response and aggregate content processed_stream = sanitize_stream( data=response.iter_content(chunk_size=None), # Pass byte iterator @@ -195,13 +197,13 @@ def _create_non_stream( yield_raw_on_error=True, raw=False ) - + # Aggregate all content content = "" for content_chunk in processed_stream: if content_chunk and isinstance(content_chunk, str): content += content_chunk - + message = ChatCompletionMessage( role="assistant", content=content @@ -254,7 +256,7 @@ def __init__(self, browser: str = "chrome"): } self.session.headers.update(self.headers) self.chat = Chat(self) - + @property def models(self): class _ModelList: @@ -270,4 +272,4 @@ def list(inner_self): max_tokens=600, stream=False ) - print(response.choices[0].message.content) \ No newline at end of file + print(response.choices[0].message.content) diff --git a/webscout/Provider/OPENAI/exaai.py b/webscout/Provider/OPENAI/exaai.py index fdfa31e3..00f8caeb 100644 --- a/webscout/Provider/OPENAI/exaai.py +++ b/webscout/Provider/OPENAI/exaai.py @@ -1,15 +1,21 @@ +import json +import re import time import uuid +from typing import Any, Dict, Generator, List, Optional, Union + import requests -import json -import re -from typing import List, Dict, Optional, Union, Generator, Any # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, ) # Attempt to import LitAgent, fallback if not available @@ -434,7 +440,7 @@ def convert_model_name(self, model: str) -> str: # ExaAI only supports O3-Mini, regardless of the input model print(f"Note: ExaAI only supports O3-Mini model. Ignoring provided model '{model}'.") return "O3-Mini" - + @property def models(self): class _ModelList: @@ -451,4 +457,4 @@ def list(inner_self): {"role": "user", "content": "Hello, how are you?"} ] ) - print(response.choices[0].message.content) \ No newline at end of file + print(response.choices[0].message.content) diff --git a/webscout/Provider/OPENAI/freeassist.py b/webscout/Provider/OPENAI/freeassist.py index c7d7a4fa..90a207c8 100644 --- a/webscout/Provider/OPENAI/freeassist.py +++ b/webscout/Provider/OPENAI/freeassist.py @@ -1,13 +1,18 @@ -import requests import json import time import uuid -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union + +import requests -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, ) try: @@ -35,7 +40,7 @@ def create( ) -> Union[ChatCompletion, Generator[ChatCompletionChunk, None, None]]: """ Create a chat completion with FreeAssist API. - + Args: model: Model identifier (e.g., 'google/gemini-2.5-flash-lite') messages: List of message dictionaries with 'role' and 'content' @@ -46,7 +51,7 @@ def create( timeout: Request timeout proxies: Proxy configuration **kwargs: Additional parameters - + Returns: ChatCompletion or Generator of ChatCompletionChunk """ @@ -56,10 +61,10 @@ def create( "anonymousUserId": str(uuid.uuid4()), "isContinuation": False } - + request_id = f"chatcmpl-{uuid.uuid4()}" created_time = int(time.time()) - + if stream: return self._create_stream(request_id, created_time, model, payload, timeout, proxies) else: @@ -80,52 +85,52 @@ def _create_stream( proxies=proxies, ) response.raise_for_status() - + prompt_tokens = 0 completion_tokens = 0 total_tokens = 0 - + for line in response.iter_lines(decode_unicode=True): if not line: continue - + if isinstance(line, bytes): try: line = line.decode("utf-8") except Exception: continue - + if line.startswith("data: "): json_str = line[6:] else: json_str = line - + if json_str == "[DONE]": break - + try: data = json.loads(json_str) - + # Extract usage if present usage_data = data.get('usage', {}) if usage_data: prompt_tokens = usage_data.get('prompt_tokens', prompt_tokens) completion_tokens = usage_data.get('completion_tokens', completion_tokens) total_tokens = usage_data.get('total_tokens', total_tokens) - + choices = data.get('choices') if not choices and choices is not None: continue - + choice_data = choices[0] if choices else {} delta_data = choice_data.get('delta', {}) finish_reason = choice_data.get('finish_reason') - + # Get content content_piece = None role = None tool_calls = None - + if delta_data: content_piece = delta_data.get('content') role = delta_data.get('role') @@ -135,24 +140,24 @@ def _create_stream( role = message_d.get("role") content_piece = message_d.get("content") tool_calls = message_d.get("tool_calls") - + if content_piece and not usage_data: completion_tokens += 1 total_tokens = prompt_tokens + completion_tokens - + delta = ChoiceDelta( content=content_piece, role=role, tool_calls=tool_calls ) - + choice = Choice( index=choice_data.get('index', 0), delta=delta, finish_reason=finish_reason, logprobs=choice_data.get('logprobs') ) - + chunk = ChatCompletionChunk( id=data.get('id', request_id), choices=[choice], @@ -160,19 +165,19 @@ def _create_stream( model=data.get('model', model), system_fingerprint=data.get('system_fingerprint') ) - + chunk.usage = { "prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens, "estimated_cost": None } - + yield chunk - + except json.JSONDecodeError: continue - + # Final chunk with finish_reason="stop" delta = ChoiceDelta(content=None, role=None, tool_calls=None) choice = Choice(index=0, delta=delta, finish_reason="stop", logprobs=None) @@ -190,7 +195,7 @@ def _create_stream( "estimated_cost": None } yield chunk - + except Exception as e: print(f"Error during FreeAssist stream request: {e}") raise IOError(f"FreeAssist request failed: {e}") from e @@ -211,75 +216,75 @@ def _create_non_stream( proxies=proxies, ) response.raise_for_status() - + full_content = "" prompt_tokens = 0 completion_tokens = 0 total_tokens = 0 response_model = model - + for line in response.iter_lines(decode_unicode=True): if not line: continue - + if isinstance(line, bytes): try: line = line.decode("utf-8") except Exception: continue - + if line.startswith("data: "): json_str = line[6:] else: json_str = line - + if json_str == "[DONE]": break - + try: data = json.loads(json_str) - + # Extract usage if present usage_data = data.get('usage', {}) if usage_data: prompt_tokens = usage_data.get('prompt_tokens', prompt_tokens) completion_tokens = usage_data.get('completion_tokens', completion_tokens) total_tokens = usage_data.get('total_tokens', total_tokens) - + choices = data.get('choices') if not choices and choices is not None: continue - + choice_data = choices[0] if choices else {} delta_data = choice_data.get('delta', {}) content = delta_data.get('content', '') - + if content: full_content += content - + # Get model from response if data.get('model'): response_model = data.get('model') except json.JSONDecodeError: continue - + message = ChatCompletionMessage( role='assistant', content=full_content ) - + choice = Choice( index=0, message=message, finish_reason='stop' ) - + usage = CompletionUsage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=total_tokens ) - + completion = ChatCompletion( id=request_id, choices=[choice], @@ -287,9 +292,9 @@ def _create_non_stream( model=response_model, usage=usage, ) - + return completion - + except Exception as e: print(f"Error during FreeAssist non-stream request: {e}") raise IOError(f"FreeAssist request failed: {e}") from e @@ -303,15 +308,15 @@ def __init__(self, client: 'FreeAssist'): class FreeAssist(OpenAICompatibleProvider): """ FreeAssist - A free OpenAI-compatible provider using FreeAssist.ai - + This provider uses the FreeAssist API which provides access to various AI models including Google's Gemini series. - + Usage: from webscout.Provider.OPENAI import FreeAssist - + client = FreeAssist() - + # Streaming for chunk in client.chat.completions.create( model="google/gemini-2.5-flash-lite", @@ -320,7 +325,7 @@ class FreeAssist(OpenAICompatibleProvider): ): if chunk.choices[0].delta.content: print(chunk.choices[0].delta.content, end="", flush=True) - + # Non-streaming response = client.chat.completions.create( model="google/gemini-2.5-flash-lite", @@ -329,7 +334,7 @@ class FreeAssist(OpenAICompatibleProvider): ) print(response.choices[0].message.content) """ - + AVAILABLE_MODELS = [ "google/gemini-2.5-flash-lite", "google/gemini-2.5-flash", @@ -345,7 +350,7 @@ def __init__( ): """ Initialize the FreeAssist provider. - + Args: browser: Browser to impersonate for fingerprinting timeout: Request timeout in seconds @@ -354,10 +359,10 @@ def __init__( self.timeout = timeout self.base_url = "https://qcpujeurnkbvwlvmylyx.supabase.co/functions/v1/chat" self.session = requests.Session() - + agent = LitAgent() fingerprint = agent.generate_fingerprint(browser) - + self.headers = { 'accept': '*/*', 'accept-language': 'en-US,en;q=0.9,en-IN;q=0.8', @@ -374,13 +379,13 @@ def __init__( 'sec-gpc': '1', 'user-agent': fingerprint.get("user_agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36 Edg/143.0.0.0") } - + self.session.headers.update(self.headers) if proxies: self.session.proxies.update(proxies) - + self.chat = Chat(self) - + @property def models(self): """Return available models.""" @@ -394,9 +399,9 @@ def list(inner_self): print("-" * 80) print("Testing FreeAssist Provider") print("-" * 80) - + client = FreeAssist() - + # Test streaming print("\n[Streaming Test]") try: @@ -412,7 +417,7 @@ def list(inner_self): print(f"\n✓ Streaming works! Response: {response_text}") except Exception as e: print(f"✗ Streaming failed: {e}") - + # Test non-streaming print("\n[Non-Streaming Test]") try: @@ -424,6 +429,6 @@ def list(inner_self): print(f"✓ Non-streaming works! Response: {response.choices[0].message.content}") except Exception as e: print(f"✗ Non-streaming failed: {e}") - + print("\n" + "-" * 80) print("Available models:", FreeAssist.AVAILABLE_MODELS) diff --git a/webscout/Provider/OPENAI/gradient.py b/webscout/Provider/OPENAI/gradient.py index c9bafcfc..7fc46508 100644 --- a/webscout/Provider/OPENAI/gradient.py +++ b/webscout/Provider/OPENAI/gradient.py @@ -8,13 +8,19 @@ import json import time import uuid +from typing import Any, Dict, Generator, List, Optional, Union + import requests -from typing import List, Dict, Optional, Union, Generator, Any -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, ) # ANSI escape codes for formatting @@ -67,7 +73,7 @@ def create( # Convert model name and get appropriate cluster mode converted_model = self._client.convert_model_name(model) actual_cluster_mode = cluster_mode or self._client.MODEL_CLUSTERS.get(converted_model, self._client.cluster_mode) - + # Build the payload - pass messages directly as the API accepts them payload = { "model": converted_model, @@ -117,32 +123,32 @@ def _create_stream( # Parse JSON response data = json.loads(decoded_line) - + # Only process "reply" type chunks chunk_type = data.get("type") if chunk_type != "reply": continue - + # Extract content - prefer "content" over "reasoningContent" reply_data = data.get("data", {}) content = reply_data.get("content") or reply_data.get("reasoningContent") - + if content: completion_tokens += count_tokens(content) - + delta = ChoiceDelta( content=content, role="assistant" if first_chunk else None ) first_chunk = False - + choice = Choice( index=0, delta=delta, finish_reason=None, logprobs=None ) - + chunk = ChatCompletionChunk( id=request_id, choices=[choice], @@ -197,7 +203,7 @@ def _create_non_stream( # Collect all chunks from streaming full_content = "" prompt_tokens = count_tokens(str(payload.get("messages", []))) - + response = self._client.session.post( self._client.base_url, headers=self._client.headers, @@ -220,12 +226,12 @@ def _create_non_stream( continue data = json.loads(decoded_line) - + # Only process "reply" type chunks chunk_type = data.get("type") if chunk_type != "reply": continue - + reply_data = data.get("data", {}) # Prefer "content" over "reasoningContent" content = reply_data.get("content") or reply_data.get("reasoningContent") @@ -285,10 +291,10 @@ def __init__(self, client: 'Gradient'): class Gradient(OpenAICompatibleProvider): """ OpenAI-compatible client for Gradient Network API. - + Gradient Network provides access to distributed GPU clusters running large language models. This provider supports real-time streaming responses. - + Note: GPT OSS 120B works on "nvidia" cluster, Qwen3 235B works on "hybrid" cluster. Cluster mode is auto-detected based on model selection. @@ -316,7 +322,7 @@ class Gradient(OpenAICompatibleProvider): "GPT OSS 120B", "Qwen3 235B", ] - + # Model to cluster mapping MODEL_CLUSTERS = { "GPT OSS 120B": "nvidia", @@ -343,7 +349,7 @@ def __init__( self.cluster_mode = cluster_mode self.enable_thinking = enable_thinking self.proxies = proxies or {} - + self.base_url = "https://chat.gradient.network/api/generate" self.session = requests.Session() diff --git a/webscout/Provider/OPENAI/groq.py b/webscout/Provider/OPENAI/groq.py index b16dda2f..803d7917 100644 --- a/webscout/Provider/OPENAI/groq.py +++ b/webscout/Provider/OPENAI/groq.py @@ -1,18 +1,22 @@ -import requests import json import time import uuid -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union + +from curl_cffi import CurlError # Import curl_cffi for improved request handling from curl_cffi.requests import Session -from curl_cffi import CurlError # Import base classes and utility structures -from .base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from .base import BaseChat, BaseCompletions, OpenAICompatibleProvider from .utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, ) # Attempt to import LitAgent, fallback if not available @@ -58,7 +62,7 @@ def create( payload["frequency_penalty"] = kwargs.pop("frequency_penalty") if "presence_penalty" in kwargs: payload["presence_penalty"] = kwargs.pop("presence_penalty") - + # Add any tools if provided if "tools" in kwargs and kwargs["tools"]: payload["tools"] = kwargs.pop("tools") @@ -84,7 +88,7 @@ def _create_stream( timeout=self._client.timeout, impersonate="chrome110" # Use impersonate for better compatibility ) - + if response.status_code != 200: raise IOError(f"Groq request failed with status code {response.status_code}: {response.text}") @@ -184,10 +188,10 @@ def _create_non_stream( timeout=self._client.timeout, impersonate="chrome110" # Use impersonate for better compatibility ) - + if response.status_code != 200: raise IOError(f"Groq request failed with status code {response.status_code}: {response.text}") - + data = response.json() choices_data = data.get('choices', []) @@ -196,10 +200,10 @@ def _create_non_stream( choices = [] for choice_d in choices_data: message_d = choice_d.get('message', {}) - + # Handle tool calls if present tool_calls = message_d.get('tool_calls') - + message = ChatCompletionMessage( role=message_d.get('role', 'assistant'), content=message_d.get('content', ''), @@ -272,26 +276,26 @@ def __init__(self, api_key: str = None, timeout: Optional[int] = 30, browser: st self.timeout = timeout self.base_url = "https://api.groq.com/openai/v1/chat/completions" self.api_key = api_key - + # Update available models from API self.update_available_models(api_key) - + # Initialize curl_cffi Session self.session = Session() - + # Set up headers with API key if provided self.headers = { "Content-Type": "application/json", } - + if api_key: self.headers["Authorization"] = f"Bearer {api_key}" - + # Try to use LitAgent for browser fingerprinting try: agent = LitAgent() fingerprint = agent.generate_fingerprint(browser) - + self.headers.update({ "Accept": fingerprint["accept"], "Accept-Encoding": "gzip, deflate, br, zstd", @@ -317,26 +321,26 @@ def __init__(self, api_key: str = None, timeout: Optional[int] = 30, browser: st "Accept-Language": "en-US,en;q=0.9", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", }) - + # Update session headers self.session.headers.update(self.headers) - + # Initialize chat interface self.chat = Chat(self) - + @classmethod def get_models(cls, api_key: str = None): """Fetch available models from Groq API. - + Args: api_key (str, optional): Groq API key. If not provided, returns default models. - + Returns: list: List of available model IDs """ if not api_key: return cls.AVAILABLE_MODELS - + try: # Use a temporary curl_cffi session for this class method temp_session = Session() @@ -344,21 +348,21 @@ def get_models(cls, api_key: str = None): "Content-Type": "application/json", "Authorization": f"Bearer {api_key}", } - + response = temp_session.get( "https://api.groq.com/openai/v1/models", headers=headers, impersonate="chrome110" # Use impersonate for fetching ) - + if response.status_code != 200: return cls.AVAILABLE_MODELS - + data = response.json() if "data" in data and isinstance(data["data"], list): return [model["id"] for model in data["data"]] return cls.AVAILABLE_MODELS - + except (CurlError, Exception): # Fallback to default models list if fetching fails return cls.AVAILABLE_MODELS diff --git a/webscout/Provider/OPENAI/hadadxyz.py b/webscout/Provider/OPENAI/hadadxyz.py index 07803748..1bb4319e 100644 --- a/webscout/Provider/OPENAI/hadadxyz.py +++ b/webscout/Provider/OPENAI/hadadxyz.py @@ -1,17 +1,20 @@ -import json import time import uuid -import re -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union from curl_cffi.requests import Session -from curl_cffi import CurlError # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, count_tokens, format_prompt + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, + format_prompt, ) try: @@ -69,10 +72,10 @@ def create( include_think_tags: bool = True, **kwargs: Any ) -> Union[ChatCompletion, Generator[ChatCompletionChunk, None, None]]: - + # Format the prompt using the utility prompt = format_prompt(messages, include_system=True) - + payload = { "tools": {}, "modelId": model, @@ -93,7 +96,7 @@ def create( request_id = f"chatcmpl-{uuid.uuid4()}" created_time = int(time.time()) - + if stream: return self._create_stream(request_id, created_time, model, payload, timeout, proxies, include_think_tags) else: @@ -105,7 +108,7 @@ def _create_stream( include_think_tags: bool = True ) -> Generator[ChatCompletionChunk, None, None]: extractor = _DeltaExtractor(include_think_tags=include_think_tags) - + try: response = self._client.session.post( self._client.api_endpoint, @@ -117,10 +120,10 @@ def _create_stream( impersonate="chrome120" ) response.raise_for_status() - + prompt_tokens = count_tokens(payload["messages"][0]["parts"][0]["text"]) completion_tokens = 0 - + from webscout.AIutel import sanitize_stream processed_stream = sanitize_stream( data=response.iter_lines(), @@ -134,7 +137,7 @@ def _create_stream( for content_chunk in processed_stream: if content_chunk and isinstance(content_chunk, str): completion_tokens += count_tokens(content_chunk) - + delta = ChoiceDelta( content=content_chunk, role="assistant" @@ -180,14 +183,14 @@ def _create_non_stream( ) -> ChatCompletion: full_content = "" prompt_tokens = count_tokens(payload["messages"][0]["parts"][0]["text"]) - + try: for chunk in self._create_stream(request_id, created_time, model, payload, timeout, proxies, include_think_tags): if chunk.choices and chunk.choices[0].delta.content: full_content += chunk.choices[0].delta.content - + completion_tokens = count_tokens(full_content) - + message = ChatCompletionMessage( role="assistant", content=full_content diff --git a/webscout/Provider/OPENAI/heckai.py b/webscout/Provider/OPENAI/heckai.py index 631813ae..2fa9e4df 100644 --- a/webscout/Provider/OPENAI/heckai.py +++ b/webscout/Provider/OPENAI/heckai.py @@ -1,19 +1,20 @@ import time import uuid +from typing import Any, Dict, Generator, List, Optional, Union + import requests -from typing import List, Dict, Optional, Union, Generator, Any from webscout.litagent import LitAgent from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( ChatCompletion, ChatCompletionChunk, - Choice, ChatCompletionMessage, + Choice, ChoiceDelta, CompletionUsage, + count_tokens, format_prompt, - count_tokens ) # ANSI escape codes for formatting diff --git a/webscout/Provider/OPENAI/huggingface.py b/webscout/Provider/OPENAI/huggingface.py index 539f1d6f..0848cb6b 100644 --- a/webscout/Provider/OPENAI/huggingface.py +++ b/webscout/Provider/OPENAI/huggingface.py @@ -1,15 +1,19 @@ import json import time import uuid -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union from curl_cffi.requests import Session -from curl_cffi import CurlError -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, ) try: @@ -45,10 +49,10 @@ def create( if top_p is not None: payload["top_p"] = top_p payload.update(kwargs) - + request_id = f"chatcmpl-{uuid.uuid4()}" created_time = int(time.time()) - + if stream: return self._create_stream(request_id, created_time, model, payload, timeout, proxies) else: @@ -70,7 +74,7 @@ def _create_stream( proxies=proxies ) response.raise_for_status() - + prompt_tokens = count_tokens([msg.get("content", "") for msg in payload.get("messages", [])]) completion_tokens = 0 total_tokens = 0 @@ -103,7 +107,7 @@ def _create_stream( if content: completion_tokens += count_tokens(content) total_tokens = prompt_tokens + completion_tokens - + delta = ChoiceDelta( content=content, role=delta_data.get('role'), @@ -131,7 +135,7 @@ def _create_stream( yield chunk except json.JSONDecodeError: continue - + # Final chunk with finish_reason="stop" delta = ChoiceDelta(content=None, role=None, tool_calls=None) choice = Choice(index=0, delta=delta, finish_reason="stop", logprobs=None) @@ -149,7 +153,7 @@ def _create_stream( "estimated_cost": None } yield chunk - + except Exception as e: raise IOError(f"HuggingFace stream request failed: {e}") from e @@ -168,10 +172,10 @@ def _create_non_stream( ) response.raise_for_status() data = response.json() - + choices_data = data.get('choices', []) usage_data = data.get('usage', {}) - + choices = [] for choice_d in choices_data: message_d = choice_d.get('message', {}) @@ -185,13 +189,13 @@ def _create_non_stream( finish_reason=choice_d.get('finish_reason', 'stop') ) choices.append(choice) - + usage = CompletionUsage( prompt_tokens=usage_data.get('prompt_tokens', 0), completion_tokens=usage_data.get('completion_tokens', 0), total_tokens=usage_data.get('total_tokens', 0) ) - + completion = ChatCompletion( id=data.get('id', request_id), choices=choices, @@ -210,7 +214,7 @@ def __init__(self, client: 'HuggingFace'): class HuggingFace(OpenAICompatibleProvider): """ OpenAI-compatible client for Hugging Face Inference API. - + Requires an API key from https://huggingface.co/settings/tokens """ required_auth = True @@ -226,7 +230,7 @@ def get_models(cls, api_key: str = None) -> List[str]: headers = {} if api_key: headers["Authorization"] = f"Bearer {api_key}" - + response = temp_session.get(url, headers=headers, timeout=10) if response.status_code == 200: data = response.json() @@ -251,7 +255,7 @@ def update_available_models(cls, api_key: str = None): def __init__(self, api_key: str, browser: str = "chrome", timeout: int = 30): if not api_key: raise ValueError("API key is required for HuggingFace") - + # Update available models from API self.update_available_models(api_key) @@ -259,10 +263,10 @@ def __init__(self, api_key: str, browser: str = "chrome", timeout: int = 30): self.timeout = timeout self.base_url = "https://router.huggingface.co/v1/chat/completions" self.session = Session() - + agent = LitAgent() fingerprint = agent.generate_fingerprint(browser) - + self.headers = { "Accept": fingerprint["accept"], "Accept-Language": fingerprint["accept_language"], @@ -291,4 +295,4 @@ def list(inner_self): # messages=[{"role": "user", "content": "Hello!"}] # ) # print(response.choices[0].message.content) - pass \ No newline at end of file + pass diff --git a/webscout/Provider/OPENAI/ibm.py b/webscout/Provider/OPENAI/ibm.py index 8cb793b5..a474a4ee 100644 --- a/webscout/Provider/OPENAI/ibm.py +++ b/webscout/Provider/OPENAI/ibm.py @@ -1,19 +1,25 @@ import json import time import uuid -from typing import List, Dict, Optional, Union, Generator, Any from datetime import datetime +from typing import Any, Dict, Generator, List, Optional, Union + +from curl_cffi import CurlError # Import curl_cffi for improved request handling from curl_cffi.requests import Session -from curl_cffi import CurlError # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, - format_prompt, count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, + format_prompt, ) # Attempt to import LitAgent, fallback if not available @@ -43,12 +49,12 @@ def create( Mimics openai.chat.completions.create """ formatted_prompt = format_prompt( - messages, - add_special_tokens=False, + messages, + add_special_tokens=False, do_continue=True, include_system=True ) - + if not formatted_prompt: raise ValueError("No valid prompt could be generated from messages") @@ -101,7 +107,7 @@ def _create_stream( timeout=self._client.timeout, impersonate="chrome110" ) - + if response.status_code in [401, 403]: # Token expired, refresh and retry once self._client.get_token() @@ -123,56 +129,56 @@ def _create_stream( for chunk in response.iter_content(chunk_size=None): if not chunk: continue - + # Decode bytes to string try: chunk_str = chunk.decode('utf-8') if isinstance(chunk, bytes) else chunk except UnicodeDecodeError: continue - + buffer += chunk_str - + # Process complete lines while '\n' in buffer: line, buffer = buffer.split('\n', 1) line = line.strip() - + if not line: continue - + # Parse SSE format: "data: {...}" if line.startswith("data:"): json_str = line[5:].strip() # Remove "data:" prefix - + # Skip [DONE] marker if json_str == "[DONE]": break - + try: # Parse JSON data = json.loads(json_str) - + # Extract content from IBM format if data.get("type") == "message.part": part = data.get("part", {}) content = part.get("content") - + if content: completion_tokens += 1 - + # Create the delta object delta = ChoiceDelta( content=content, role="assistant" ) - + # Create the choice object choice = Choice( index=0, delta=delta, finish_reason=None ) - + # Create the chunk object chunk = ChatCompletionChunk( id=request_id, @@ -181,9 +187,9 @@ def _create_stream( model=model, system_fingerprint=None ) - + yield chunk - + except json.JSONDecodeError: # Skip malformed JSON lines continue @@ -309,7 +315,7 @@ def get_token(self) -> str: def __init__(self, api_key: str = None, timeout: Optional[int] = 30, browser: str = "chrome"): """ Initialize IBM client. - + Args: api_key: Not required for IBM Granite Playground (uses dynamic bearer token) timeout: Request timeout in seconds @@ -317,15 +323,15 @@ def __init__(self, api_key: str = None, timeout: Optional[int] = 30, browser: st """ self.timeout = timeout self.base_url = "https://d1eh1ubv87xmm5.cloudfront.net/granite/playground/api/v1/acp/runs" - + # Initialize curl_cffi Session self.session = Session() - + # Initialize LitAgent for browser fingerprinting try: agent = LitAgent() fingerprint = agent.generate_fingerprint(browser) - + self.headers = { "Accept": "text/event-stream", "Accept-Language": fingerprint.get("accept_language", "en-US,en;q=0.9"), @@ -362,23 +368,23 @@ def __init__(self, api_key: str = None, timeout: Optional[int] = 30, browser: st "Sec-CH-UA-Mobile": "?0", "Sec-CH-UA-Platform": '"Windows"', } - + # Update session headers self.session.headers.update(self.headers) # Fetch initial token self.get_token() - + # Initialize chat interface self.chat = Chat(self) @classmethod def get_models(cls, api_key: str = None): """Get available models. - + Args: api_key: Not used for IBM (kept for compatibility) - + Returns: list: List of available model IDs """ @@ -397,7 +403,7 @@ def list(inner_self): if __name__ == "__main__": # Test the IBM client client = IBM() - + # Test streaming print("Testing streaming:") response = client.chat.completions.create( @@ -407,12 +413,12 @@ def list(inner_self): ], stream=True ) - + for chunk in response: if chunk.choices and chunk.choices[0].delta.content: print(chunk.choices[0].delta.content, end="", flush=True) print("\n") - + # Test non-streaming print("Testing non-streaming:") response = client.chat.completions.create( @@ -422,5 +428,5 @@ def list(inner_self): ], stream=False ) - + print(response.choices[0].message.content) diff --git a/webscout/Provider/OPENAI/llmchat.py b/webscout/Provider/OPENAI/llmchat.py index f7613bfd..65f135a0 100644 --- a/webscout/Provider/OPENAI/llmchat.py +++ b/webscout/Provider/OPENAI/llmchat.py @@ -1,21 +1,23 @@ +import json import time import uuid -import json -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union from curl_cffi.requests import Session -from curl_cffi import CurlError +# Import LitAgent for user agent generation # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, ) -from webscout.Provider.OPENAI.utils import format_prompt -# Import LitAgent for user agent generation -from webscout.litagent import LitAgent class Completions(BaseCompletions): def __init__(self, client: 'LLMChat'): @@ -34,7 +36,7 @@ def create( proxies: Optional[dict] = None, **kwargs: Any ) -> Union[ChatCompletion, Generator[ChatCompletionChunk, None, None]]: - + # In this case, we pass messages directly to the API request_id = f"chatcmpl-{uuid.uuid4()}" created_time = int(time.time()) @@ -57,7 +59,7 @@ def _create_streaming( try: prompt_tokens = count_tokens(json.dumps(messages)) completion_tokens = 0 - + url = f"{self._client.api_endpoint}?model={model}" payload = { "messages": messages, @@ -82,14 +84,14 @@ def _create_streaming( data_str = line[6:] if data_str.strip() == "[DONE]": break - + try: data = json.loads(data_str) content = data.get('response', '') if content: full_content += content completion_tokens += 1 - + delta = ChoiceDelta(content=content, role="assistant") choice = Choice(index=0, delta=delta, finish_reason=None) chunk = ChatCompletionChunk( @@ -128,11 +130,11 @@ def _create_non_streaming( try: full_content = "" prompt_tokens = count_tokens(json.dumps(messages)) - + for chunk in self._create_streaming(request_id, created_time, model, messages, max_tokens, timeout, proxies): if chunk.choices[0].delta.content: full_content += chunk.choices[0].delta.content - + message = ChatCompletionMessage(role="assistant", content=full_content) choice = Choice(index=0, message=message, finish_reason="stop") usage = CompletionUsage( @@ -140,7 +142,7 @@ def _create_non_streaming( completion_tokens=count_tokens(full_content), total_tokens=prompt_tokens + count_tokens(full_content) ) - + return ChatCompletion( id=request_id, choices=[choice], @@ -213,7 +215,7 @@ def __init__(self, proxies: dict = {}, timeout: int = 30): self.api_endpoint = "https://llmchat.in/inference/stream" self.proxies = proxies self.session.proxies = proxies - + self.headers = { "Content-Type": "application/json", "Accept": "*/*", diff --git a/webscout/Provider/OPENAI/llmchatco.py b/webscout/Provider/OPENAI/llmchatco.py index efee0b5b..b80beea4 100644 --- a/webscout/Provider/OPENAI/llmchatco.py +++ b/webscout/Provider/OPENAI/llmchatco.py @@ -1,14 +1,21 @@ +import json import time import uuid +from typing import Any, Dict, Generator, List, Optional, Union + import requests -import json -from typing import List, Dict, Optional, Union, Generator, Any # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, get_last_user_message, get_system_prompt, format_prompt # Import format_prompt + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, # Import format_prompt + Choice, + ChoiceDelta, + CompletionUsage, + get_last_user_message, + get_system_prompt, ) # Attempt to import LitAgent, fallback if not available @@ -52,7 +59,7 @@ def create( # Determine the effective system prompt effective_system_prompt = system_prompt # Use the provided system_prompt or its default - message_list_system_prompt = get_system_prompt(messages) + get_system_prompt(messages) # If a system prompt is also in messages, the explicit one takes precedence. # We'll use the effective_system_prompt determined above. @@ -344,4 +351,4 @@ def list(inner_self): messages=[{"role": "user", "content": "Hello, how are you?"}], stream=False ) - print(response.choices[0].message.content) \ No newline at end of file + print(response.choices[0].message.content) diff --git a/webscout/Provider/OPENAI/meta.py b/webscout/Provider/OPENAI/meta.py index 9b853e7f..888f424c 100644 --- a/webscout/Provider/OPENAI/meta.py +++ b/webscout/Provider/OPENAI/meta.py @@ -7,15 +7,19 @@ import time import urllib import uuid -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union -from curl_cffi.requests import Session from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, ) try: @@ -176,7 +180,7 @@ def _create_stream( try: json_line = json.loads(line) message_text = format_response(json_line) - + if not message_text: continue @@ -304,7 +308,7 @@ def _create_non_stream( raw_response = response.text last_response = None full_message = "" - + for line in raw_response.split("\n"): try: json_line = json.loads(line) @@ -363,7 +367,7 @@ class Meta(OpenAICompatibleProvider): No API key required - uses web authentication. """ required_auth = False - + AVAILABLE_MODELS = [ "meta-ai", "llama-3" @@ -379,7 +383,7 @@ def __init__( ): """ Initialize the Meta AI OpenAI-compatible client. - + Args: fb_email: Optional Facebook email for authenticated access fb_password: Optional Facebook password for authenticated access @@ -391,30 +395,30 @@ def __init__( self.fb_password = fb_password self.timeout = timeout self.proxies = proxies or {} - + self.session = Session() if LitAgent: agent = LitAgent() self.session.headers.update({ "user-agent": agent.random() }) - + self.access_token = None self.is_authed = fb_password is not None and fb_email is not None self.cookies = self._get_cookies() self.external_conversation_id = None self.offline_threading_id = None - + if proxies: self.session.proxies = proxies - + # Initialize chat interface self.chat = Chat(self) def _get_cookies(self) -> dict: """Extracts necessary cookies from the Meta AI main page.""" headers = {} - + # Import Facebook login if needed if self.fb_email is not None and self.fb_password is not None: try: @@ -513,7 +517,7 @@ def list(inner_self): # Example usage - no API key required client = Meta() print(f"Available models: {client.models.list()}") - + # Test non-streaming response = client.chat.completions.create( model="meta-ai", @@ -521,7 +525,7 @@ def list(inner_self): stream=False ) print(f"Response: {response.choices[0].message.content}") - + # Test streaming print("\nStreaming response:") for chunk in client.chat.completions.create( diff --git a/webscout/Provider/OPENAI/netwrck.py b/webscout/Provider/OPENAI/netwrck.py index 9df1d11d..4bb81b9f 100644 --- a/webscout/Provider/OPENAI/netwrck.py +++ b/webscout/Provider/OPENAI/netwrck.py @@ -1,21 +1,22 @@ +import json import time import uuid +from typing import Any, Dict, Generator, List, Optional, Union + import requests -import json -from typing import List, Dict, Optional, Union, Generator, Any from webscout.litagent import LitAgent from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( ChatCompletion, ChatCompletionChunk, - Choice, ChatCompletionMessage, + Choice, ChoiceDelta, CompletionUsage, + count_tokens, format_prompt, get_system_prompt, - count_tokens ) # ANSI escape codes for formatting @@ -47,8 +48,8 @@ def create( # Format the messages using the format_prompt utility # This creates a conversation in the format: "User: message\nAssistant: response\nUser: message\nAssistant:" formatted_prompt = format_prompt(messages, add_special_tokens=True, do_continue=True) - - + + # Prepare the payload for Netwrck API payload = { "query": formatted_prompt, @@ -87,7 +88,7 @@ def _create_stream( for line in response.iter_lines(): if not line: continue - + try: decoded_line = line.decode('utf-8').strip('"') if decoded_line: @@ -95,18 +96,18 @@ def _create_stream( formatted_content = self._client.format_text(decoded_line) streaming_text += formatted_content completion_tokens += count_tokens(formatted_content) - + # Create a delta object for this chunk delta = ChoiceDelta(content=formatted_content) choice = Choice(index=0, delta=delta, finish_reason=None) - + chunk = ChatCompletionChunk( id=request_id, choices=[choice], created=created_time, model=model, ) - + yield chunk except Exception: continue @@ -114,14 +115,14 @@ def _create_stream( # Final chunk with finish_reason delta = ChoiceDelta(content=None) choice = Choice(index=0, delta=delta, finish_reason="stop") - + chunk = ChatCompletionChunk( id=request_id, choices=[choice], created=created_time, model=model, ) - + yield chunk except requests.exceptions.RequestException as e: @@ -140,7 +141,7 @@ def _create_non_stream( proxies=proxies or getattr(self._client, "proxies", None) ) response.raise_for_status() - + # Process the response raw_response = response.text.strip('"') # Format the full response using the client's formatter @@ -150,26 +151,26 @@ def _create_non_stream( prompt_tokens = count_tokens(payload.get("query", "")) completion_tokens = count_tokens(full_response) total_tokens = prompt_tokens + completion_tokens - + usage = CompletionUsage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=total_tokens ) - + # Create the message object message = ChatCompletionMessage( role="assistant", content=full_response ) - + # Create the choice object choice = Choice( index=0, message=message, finish_reason="stop" ) - + # Create the completion object completion = ChatCompletion( id=request_id, @@ -178,7 +179,7 @@ def _create_non_stream( model=model, usage=usage, ) - + return completion except Exception as e: @@ -236,10 +237,10 @@ def __init__( self.temperature = temperature self.top_p = top_p self.system_prompt = system_prompt - + # Initialize LitAgent for user agent generation agent = LitAgent() - + self.headers = { 'authority': 'netwrck.com', 'accept': '*/*', @@ -249,10 +250,10 @@ def __init__( 'referer': 'https://netwrck.com/', 'user-agent': agent.random() } - + self.session = requests.Session() self.session.headers.update(self.headers) - + # Initialize the chat interface self.chat = Chat(self) @@ -300,12 +301,12 @@ def convert_model_name(self, model: str) -> str: """ if model in self.AVAILABLE_MODELS: return model - + # Try to find a matching model for available_model in self.AVAILABLE_MODELS: if model.lower() in available_model.lower(): return available_model - + # Default to DeepSeek if no match print(f"{BOLD}Warning: Model '{model}' not found, using default model 'deepseek/deepseek-r1'{RESET}") return "deepseek/deepseek-r1" @@ -342,7 +343,7 @@ def list(inner_self): ], stream=False ) - + if response and response.choices and response.choices[0].message.content: status = "✓" # Truncate response if too long diff --git a/webscout/Provider/OPENAI/nvidia.py b/webscout/Provider/OPENAI/nvidia.py index a9a608dd..75a02976 100644 --- a/webscout/Provider/OPENAI/nvidia.py +++ b/webscout/Provider/OPENAI/nvidia.py @@ -1,15 +1,19 @@ import json import time import uuid -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union from curl_cffi.requests import Session -from curl_cffi import CurlError -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, ) try: @@ -45,10 +49,10 @@ def create( if top_p is not None: payload["top_p"] = top_p payload.update(kwargs) - + request_id = f"chatcmpl-{uuid.uuid4()}" created_time = int(time.time()) - + if stream: return self._create_stream(request_id, created_time, model, payload, timeout, proxies) else: @@ -69,7 +73,7 @@ def _create_stream( impersonate="chrome120" ) response.raise_for_status() - + prompt_tokens = count_tokens([msg.get("content", "") for msg in payload.get("messages", [])]) completion_tokens = 0 total_tokens = 0 @@ -101,7 +105,7 @@ def _create_stream( if content: completion_tokens += count_tokens(content) total_tokens = prompt_tokens + completion_tokens - + delta = ChoiceDelta( content=content, role=delta_data.get('role'), @@ -129,7 +133,7 @@ def _create_stream( yield chunk except json.JSONDecodeError: continue - + # Final chunk with finish_reason="stop" delta = ChoiceDelta(content=None, role=None, tool_calls=None) choice = Choice(index=0, delta=delta, finish_reason="stop", logprobs=None) @@ -147,7 +151,7 @@ def _create_stream( "estimated_cost": None } yield chunk - + except Exception as e: raise IOError(f"Nvidia stream request failed: {e}") from e @@ -166,10 +170,10 @@ def _create_non_stream( ) response.raise_for_status() data = response.json() - + choices_data = data.get('choices', []) usage_data = data.get('usage', {}) - + choices = [] for choice_d in choices_data: message_d = choice_d.get('message', {}) @@ -183,13 +187,13 @@ def _create_non_stream( finish_reason=choice_d.get('finish_reason', 'stop') ) choices.append(choice) - + usage = CompletionUsage( prompt_tokens=usage_data.get('prompt_tokens', 0), completion_tokens=usage_data.get('completion_tokens', 0), total_tokens=usage_data.get('total_tokens', 0) ) - + completion = ChatCompletion( id=data.get('id', request_id), choices=choices, @@ -222,7 +226,7 @@ def get_models(cls, api_key: str = None) -> List[str]: headers = {} if api_key: headers["Authorization"] = f"Bearer {api_key}" - + response = temp_session.get(url, headers=headers, timeout=10) if response.status_code == 200: data = response.json() @@ -246,7 +250,7 @@ def update_available_models(cls, api_key: str = None): def __init__(self, api_key: str, browser: str = "chrome", timeout: int = 30): if not api_key: raise ValueError("API key is required for Nvidia") - + # Update available models from API self.update_available_models(api_key) @@ -254,10 +258,10 @@ def __init__(self, api_key: str, browser: str = "chrome", timeout: int = 30): self.timeout = timeout self.base_url = "https://integrate.api.nvidia.com/v1/chat/completions" self.session = Session() - + agent = LitAgent() fingerprint = agent.generate_fingerprint(browser) - + self.headers = { "Accept": "application/json", "Accept-Language": fingerprint["accept_language"], diff --git a/webscout/Provider/OPENAI/oivscode.py b/webscout/Provider/OPENAI/oivscode.py index eb2fbc24..17be27f1 100644 --- a/webscout/Provider/OPENAI/oivscode.py +++ b/webscout/Provider/OPENAI/oivscode.py @@ -1,18 +1,22 @@ +import json import random import secrets -from regex import F -import requests -import json +import string import time import uuid -import string -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union + +import requests # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, ) # --- oivscode Client --- diff --git a/webscout/Provider/OPENAI/sambanova.py b/webscout/Provider/OPENAI/sambanova.py index edf07937..742cbad1 100644 --- a/webscout/Provider/OPENAI/sambanova.py +++ b/webscout/Provider/OPENAI/sambanova.py @@ -6,15 +6,19 @@ import json import time import uuid -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union -from curl_cffi.requests import Session from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, ) try: @@ -249,35 +253,35 @@ class Sambanova(OpenAICompatibleProvider): Requires API key from https://cloud.sambanova.ai/ """ required_auth = True - + AVAILABLE_MODELS = [] @classmethod - def get_models(cls, api_key: str = None) -> List[str]: + def get_models(cls, api_key: Optional[str] = None) -> List[str]: """Fetch available models from Sambanova API.""" if not api_key: raise ValueError("API key is required to fetch models.") - + try: temp_session = Session() headers = { "Content-Type": "application/json", "Authorization": f"Bearer {api_key}", } - + response = temp_session.get( "https://api.sambanova.ai/v1/models", headers=headers, impersonate="chrome120" ) - + if response.status_code == 200: data = response.json() if "data" in data and isinstance(data["data"], list): return [model['id'] for model in data['data'] if 'id' in model] - + return cls.AVAILABLE_MODELS - + except Exception: return cls.AVAILABLE_MODELS @@ -289,7 +293,7 @@ def __init__( ): """ Initialize the Sambanova OpenAI-compatible client. - + Args: api_key: Your Sambanova API key (required) timeout: Request timeout in seconds @@ -297,13 +301,13 @@ def __init__( """ if not api_key: raise ValueError("API key is required for Sambanova") - + self.api_key = api_key self.timeout = timeout self.base_url = "https://api.sambanova.ai/v1/chat/completions" - + self.session = Session() - + # Generate browser fingerprint if LitAgent: agent = LitAgent() @@ -325,17 +329,17 @@ def __init__( "Content-Type": "application/json", "Authorization": f"Bearer {api_key}", } - + self.session.headers.update(self.headers) - + # Update models list dynamically self.update_available_models(api_key) - + # Initialize chat interface self.chat = Chat(self) @classmethod - def update_available_models(cls, api_key: str = None): + def update_available_models(cls, api_key: Optional[str] = None): """Update the available models list from Sambanova API.""" if api_key: models = cls.get_models(api_key) @@ -353,14 +357,14 @@ def list(inner_self): if __name__ == "__main__": # Example usage - requires API key import os - + api_key = os.environ.get("SAMBANOVA_API_KEY", "") if not api_key: print("Set SAMBANOVA_API_KEY environment variable to test") else: client = Sambanova(api_key=api_key) print(f"Available models: {client.models.list()}") - + # Test non-streaming response = client.chat.completions.create( model="Meta-Llama-3.1-8B-Instruct", @@ -368,16 +372,23 @@ def list(inner_self): max_tokens=100, stream=False ) - print(f"Response: {response.choices[0].message.content}") - + if isinstance(response, ChatCompletion): + print(f"Response: {response.choices[0].message.content}") + else: + print(f"Response: {response}") + # Test streaming print("\nStreaming response:") - for chunk in client.chat.completions.create( + stream_resp = client.chat.completions.create( model="Meta-Llama-3.1-8B-Instruct", messages=[{"role": "user", "content": "Say hello briefly"}], max_tokens=100, stream=True - ): - if chunk.choices[0].delta.content: - print(chunk.choices[0].delta.content, end="", flush=True) + ) + if hasattr(stream_resp, "__iter__") and not isinstance(stream_resp, (str, bytes, ChatCompletion)): + for chunk in stream_resp: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="", flush=True) + else: + print(stream_resp) print() diff --git a/webscout/Provider/OPENAI/sonus.py b/webscout/Provider/OPENAI/sonus.py index a94f52e2..006d2559 100644 --- a/webscout/Provider/OPENAI/sonus.py +++ b/webscout/Provider/OPENAI/sonus.py @@ -1,20 +1,21 @@ +import json import time import uuid +from typing import Any, Dict, Generator, List, Optional, Union + import requests -import json -from typing import List, Dict, Optional, Union, Generator, Any from webscout.litagent import LitAgent from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( ChatCompletion, ChatCompletionChunk, - Choice, ChatCompletionMessage, + Choice, ChoiceDelta, CompletionUsage, + count_tokens, format_prompt, - count_tokens ) # ANSI escape codes for formatting @@ -47,7 +48,7 @@ def create( # This creates a conversation in the format: "User: message\nAssistant: response\nUser: message\nAssistant:" # SonusAI works better with a properly formatted conversation question = format_prompt(messages, add_special_tokens=True, do_continue=True) - + # Extract reasoning parameter if provided reasoning = kwargs.get('reasoning', False) @@ -88,30 +89,30 @@ def _create_stream( for line in response.iter_lines(): if not line: continue - + try: # Decode the line and remove 'data: ' prefix if present line_text = line.decode('utf-8') if line_text.startswith('data: '): line_text = line_text[6:] - + data = json.loads(line_text) if "content" in data: content = data["content"] streaming_text += content completion_tokens += count_tokens(content) - + # Create a delta object for this chunk delta = ChoiceDelta(content=content) choice = Choice(index=0, delta=delta, finish_reason=None) - + chunk = ChatCompletionChunk( id=request_id, choices=[choice], created=created_time, model=model, ) - + yield chunk except (json.JSONDecodeError, UnicodeDecodeError): continue @@ -119,14 +120,14 @@ def _create_stream( # Final chunk with finish_reason delta = ChoiceDelta(content=None) choice = Choice(index=0, delta=delta, finish_reason="stop") - + chunk = ChatCompletionChunk( id=request_id, choices=[choice], created=created_time, model=model, ) - + yield chunk except requests.exceptions.RequestException as e: @@ -163,26 +164,26 @@ def _create_non_stream( prompt_tokens = count_tokens(files.get('message', ['',''])[1]) completion_tokens = count_tokens(full_response) total_tokens = prompt_tokens + completion_tokens - + usage = CompletionUsage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=total_tokens ) - + # Create the message object message = ChatCompletionMessage( role="assistant", content=full_response ) - + # Create the choice object choice = Choice( index=0, message=message, finish_reason="stop" ) - + # Create the completion object completion = ChatCompletion( id=request_id, @@ -191,7 +192,7 @@ def _create_non_stream( model=model, usage=usage, ) - + return completion except Exception as e: @@ -233,7 +234,7 @@ def __init__( """ self.timeout = timeout self.url = "https://chat.sonus.ai/chat.php" - + # Headers for the request agent = LitAgent() self.headers = { @@ -243,10 +244,10 @@ def __init__( 'Referer': 'https://chat.sonus.ai/', 'User-Agent': agent.random() } - + self.session = requests.Session() self.session.headers.update(self.headers) - + # Initialize the chat interface self.chat = Chat(self) @@ -256,12 +257,12 @@ def convert_model_name(self, model: str) -> str: """ if model in self.AVAILABLE_MODELS: return model - + # Try to find a matching model for available_model in self.AVAILABLE_MODELS: if model.lower() in available_model.lower(): return available_model - + # Default to pro if no match print(f"{BOLD}Warning: Model '{model}' not found, using default model 'pro'{RESET}") return "pro" @@ -294,8 +295,8 @@ def list(inner_self): ], stream=False ) - - if response and response.choices and response.choices[0].message.content: + + if isinstance(response, ChatCompletion) and response.choices and response.choices[0].message.content: status = "✓" # Truncate response if too long display_text = response.choices[0].message.content.strip() diff --git a/webscout/Provider/OPENAI/textpollinations.py b/webscout/Provider/OPENAI/textpollinations.py index c0a77abd..364181e4 100644 --- a/webscout/Provider/OPENAI/textpollinations.py +++ b/webscout/Provider/OPENAI/textpollinations.py @@ -3,18 +3,26 @@ https://text.pollinations.ai/openai """ +import json import time import uuid +from typing import Any, Dict, Generator, List, Optional, Union + import requests -import json -from typing import List, Dict, Optional, Union, Generator, Any -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.litagent import LitAgent +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, ToolCall, ToolFunction, count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + ToolCall, + ToolFunction, + count_tokens, ) -from webscout.litagent import LitAgent BOLD = "\033[1m" RED = "\033[91m" @@ -254,7 +262,7 @@ class TextPollinations(OpenAICompatibleProvider): Provides free access to various models including GPT variants and open-source models. """ required_auth = False - + AVAILABLE_MODELS = ["openai", "mistral", "p1", "unity"] @classmethod @@ -273,7 +281,7 @@ def get_models(cls, api_key: str = None) -> List[str]: data = response.json() if isinstance(data, list): return [model.get("name") for model in data if isinstance(model, dict) and "name" in model] - + return cls.AVAILABLE_MODELS except Exception: @@ -339,9 +347,9 @@ def list(inner_self): response = client.chat.completions.create( model=model_to_use, messages=[{"role": "user", "content": "Hello!"}] - ) + ) print(response.choices[0].message.content) except Exception as e: print(f"Error testing model: {e}") else: - print("No models available.") \ No newline at end of file + print("No models available.") diff --git a/webscout/Provider/OPENAI/toolbaz.py b/webscout/Provider/OPENAI/toolbaz.py index 03a878c3..a9a8366c 100644 --- a/webscout/Provider/OPENAI/toolbaz.py +++ b/webscout/Provider/OPENAI/toolbaz.py @@ -1,25 +1,25 @@ -import time -import uuid import base64 import json import random -import string import re -import cloudscraper +import string +import time +import uuid from datetime import datetime -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union + +import cloudscraper from webscout.litagent import LitAgent from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( ChatCompletion, ChatCompletionChunk, - Choice, ChatCompletionMessage, + Choice, ChoiceDelta, CompletionUsage, format_prompt, - get_system_prompt ) # ANSI escape codes for formatting @@ -50,7 +50,7 @@ def create( """ # Format the messages using the format_prompt utility formatted_prompt = format_prompt(messages, add_special_tokens=True, do_continue=True) - + # Get authentication token auth = self._client.get_auth() if not auth: @@ -97,38 +97,38 @@ def _handle_streaming_response( buffer = "" tag_start = "[model:" streaming_text = "" - + for chunk in resp.iter_content(chunk_size=1): if chunk: text = chunk.decode(errors="ignore") buffer += text - + # Remove all complete [model: ...] tags in buffer while True: match = re.search(r"\[model:.*?\]", buffer) if not match: break buffer = buffer[:match.start()] + buffer[match.end():] - + # Only yield up to the last possible start of a tag last_tag = buffer.rfind(tag_start) if last_tag == -1 or last_tag + len(tag_start) > len(buffer): if buffer: streaming_text += buffer - + # Create the delta object delta = ChoiceDelta( content=buffer, role="assistant" ) - + # Create the choice object choice = Choice( index=0, delta=delta, finish_reason=None ) - + # Create the chunk object chunk = ChatCompletionChunk( id=request_id, @@ -136,26 +136,26 @@ def _handle_streaming_response( created=created_time, model=model ) - + yield chunk buffer = "" else: if buffer[:last_tag]: streaming_text += buffer[:last_tag] - + # Create the delta object delta = ChoiceDelta( content=buffer[:last_tag], role="assistant" ) - + # Create the choice object choice = Choice( index=0, delta=delta, finish_reason=None ) - + # Create the chunk object chunk = ChatCompletionChunk( id=request_id, @@ -163,10 +163,10 @@ def _handle_streaming_response( created=created_time, model=model ) - + yield chunk buffer = buffer[last_tag:] - + # Remove any remaining [model: ...] tag in the buffer buffer = re.sub(r"\[model:.*?\]", "", buffer) if buffer: @@ -175,14 +175,14 @@ def _handle_streaming_response( content=buffer, role="assistant" ) - + # Create the choice object choice = Choice( index=0, delta=delta, finish_reason="stop" ) - + # Create the chunk object chunk = ChatCompletionChunk( id=request_id, @@ -190,30 +190,30 @@ def _handle_streaming_response( created=created_time, model=model ) - + yield chunk - + # Final chunk with finish_reason delta = ChoiceDelta( content=None, role=None ) - + choice = Choice( index=0, delta=delta, finish_reason="stop" ) - + chunk = ChatCompletionChunk( id=request_id, choices=[choice], created=created_time, model=model ) - + yield chunk - + except Exception as e: print(f"{RED}Error during Toolbaz streaming request: {e}{RESET}") raise IOError(f"Toolbaz streaming request failed: {e}") from e @@ -246,17 +246,17 @@ def _handle_non_streaming_response( role="assistant", content=text ) - + # Create the choice object choice = Choice( index=0, message=message, finish_reason="stop" ) - + # Usage data is not provided by this API in a standard way, set to 0 usage = CompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0) - + # Create the completion object completion = ChatCompletion( id=request_id, @@ -265,9 +265,9 @@ def _handle_non_streaming_response( model=model, usage=usage ) - + return completion - + except Exception as e: print(f"{RED}Error during Toolbaz non-stream request: {e}{RESET}") raise IOError(f"Toolbaz request failed: {e}") from e @@ -305,7 +305,7 @@ class Toolbaz(OpenAICompatibleProvider): "grok-4-fast", "grok-4.1-fast", - + "toolbaz-v4.5-fast", "toolbaz_v4", "toolbaz_v3.5_pro", @@ -316,7 +316,7 @@ class Toolbaz(OpenAICompatibleProvider): "Llama-4-Maverick", "Llama-3.3-70B", - + "mixtral_8x22b", "L3-70B-Euryale-v2.1", "midnight-rose", @@ -340,10 +340,10 @@ def __init__( """ self.timeout = timeout self.proxies = proxies - + # Initialize session with cloudscraper self.session = cloudscraper.create_scraper() - + # Set up headers self.session.headers.update({ **LitAgent().generate_fingerprint(browser=browser), diff --git a/webscout/Provider/OPENAI/typefully.py b/webscout/Provider/OPENAI/typefully.py index dfe45b99..37156586 100644 --- a/webscout/Provider/OPENAI/typefully.py +++ b/webscout/Provider/OPENAI/typefully.py @@ -1,30 +1,30 @@ +import json import time import uuid -import json -import re -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union + +from curl_cffi import CurlError + +# Import curl_cffi for better request handling +from curl_cffi.requests import Session + +# Import LitAgent for browser fingerprinting +from webscout.litagent import LitAgent # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, + count_tokens, # Import format_prompt, get_system_prompt and count_tokens format_prompt, get_system_prompt, - count_tokens, # Import format_prompt, get_system_prompt and count_tokens ) -# Import LitAgent for browser fingerprinting -from webscout.litagent import LitAgent - -# Import curl_cffi for better request handling -from curl_cffi.requests import Session -from curl_cffi import CurlError - # ANSI escape codes for formatting BOLD = "\033[1m" RED = "\033[91m" @@ -324,8 +324,6 @@ def __init__( @staticmethod def _typefully_extractor(chunk) -> str: """Extracts content from Typefully AI SSE format.""" - import re - import json # Handle parsed JSON objects (when to_json=True) if isinstance(chunk, dict): @@ -415,4 +413,7 @@ def list(self): max_tokens=150, ) - print(f"{BOLD}Response:{RESET} {response.choices[0].message.content}") + if isinstance(response, ChatCompletion): + print(f"{BOLD}Response:{RESET} {response.choices[0].message.content}") + else: + print(f"{BOLD}Response:{RESET} {response}") diff --git a/webscout/Provider/OPENAI/typliai.py b/webscout/Provider/OPENAI/typliai.py index 9ea169b3..e55180e0 100644 --- a/webscout/Provider/OPENAI/typliai.py +++ b/webscout/Provider/OPENAI/typliai.py @@ -2,22 +2,25 @@ TypliAI OpenAI-compatible provider. """ -import json -import time -import uuid import random import string -from typing import List, Dict, Optional, Union, Generator, Any +import time +import uuid +from typing import Any, Dict, Generator, List, Optional, Union from curl_cffi.requests import Session -from curl_cffi import CurlError -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.AIutel import sanitize_stream +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, format_prompt + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + format_prompt, ) -from webscout.AIutel import sanitize_stream try: from webscout.litagent import LitAgent @@ -107,10 +110,10 @@ def _create_stream( full_response = "" completion_tokens = 0 - + # Use chunks from iter_content data_generator = response.iter_content(chunk_size=None) - + processed_stream = sanitize_stream( data=data_generator, intro_value="data: ", @@ -123,7 +126,7 @@ def _create_stream( if content and isinstance(content, str): full_response += content completion_tokens += 1 - + delta = ChoiceDelta( content=content, role="assistant" if completion_tokens == 1 else None @@ -171,7 +174,7 @@ def _create_non_stream( for chunk in self._create_stream(request_id, created_time, model, prompt, timeout, proxies): if chunk.choices[0].delta.content: full_content += chunk.choices[0].delta.content - + message = ChatCompletionMessage( role="assistant", content=full_content @@ -206,12 +209,12 @@ class TypliAI(OpenAICompatibleProvider): OpenAI-compatible client for TypliAI. """ required_auth = False - + AVAILABLE_MODELS = [ - "openai/gpt-4.1-mini", + "openai/gpt-4.1-mini", "openai/gpt-4.1", - "openai/gpt-5-mini", - "openai/gpt-5.2", + "openai/gpt-5-mini", + "openai/gpt-5.2", "openai/gpt-5.2-pro", "google/gemini-2.5-flash", "anthropic/claude-haiku-4-5", @@ -222,17 +225,17 @@ class TypliAI(OpenAICompatibleProvider): def __init__( self, timeout: int = 60, - proxies: dict = None, + proxies: Optional[dict] = None, browser: str = "chrome" ): self.timeout = timeout self.proxies = proxies or {} self.api_endpoint = "https://typli.ai/api/generators/chat" - + self.session = Session() self.agent = LitAgent() if LitAgent else None user_agent = self.agent.random() if self.agent else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" - + self.headers = { 'accept': '/', 'accept-language': 'en-US,en;q=0.9,en-IN;q=0.8', @@ -250,11 +253,11 @@ def __init__( 'sec-gpc': '1', 'user-agent': user_agent, } - + self.session.headers.update(self.headers) if proxies: self.session.proxies = proxies - + self.chat = Chat(self) @property @@ -268,7 +271,7 @@ def list(inner_self): if __name__ == "__main__": client = TypliAI() print(f"Available models: {client.models.list()}") - + # Test non-streaming print("\n=== Testing Non-Streaming ===") response = client.chat.completions.create( @@ -276,15 +279,22 @@ def list(inner_self): messages=[{"role": "user", "content": "Hello! How are you?"}], stream=False ) - print(f"Response: {response.choices[0].message.content}") - + if isinstance(response, ChatCompletion): + print(f"Response: {response.choices[0].message.content}") + else: + print(f"Response: {response}") + # Test streaming print("\n=== Testing Streaming ===") - for chunk in client.chat.completions.create( + stream_resp = client.chat.completions.create( model="openai/gpt-4.1-mini", messages=[{"role": "user", "content": "Tell me a joke"}], stream=True - ): - if chunk.choices[0].delta.content: - print(chunk.choices[0].delta.content, end="", flush=True) + ) + if hasattr(stream_resp, "__iter__") and not isinstance(stream_resp, (str, bytes, ChatCompletion)): + for chunk in stream_resp: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="", flush=True) + else: + print(stream_resp) print() diff --git a/webscout/Provider/OPENAI/utils.py b/webscout/Provider/OPENAI/utils.py index 1e760fdb..25f8e793 100644 --- a/webscout/Provider/OPENAI/utils.py +++ b/webscout/Provider/OPENAI/utils.py @@ -1,10 +1,9 @@ -from typing import List, Dict, Optional, Any -from enum import Enum import time import uuid -from webscout.Provider.OPENAI.pydantic_imports import ( - BaseModel, Field, StrictStr, StrictInt -) +from enum import Enum +from typing import Any, Dict, List, Optional + +from webscout.Provider.OPENAI.pydantic_imports import BaseModel, Field, StrictInt, StrictStr # --- OpenAI Response Structure Mimics --- # Moved here for reusability across different OpenAI-compatible providers @@ -297,7 +296,7 @@ def count_tokens(text_or_messages: Any) -> int: Returns: int: Number of tokens. """ - import tiktoken # type: ignore + import tiktoken # type: ignore if isinstance(text_or_messages, str): enc = tiktoken.encoding_for_model("gpt-4o") return len(enc.encode(text_or_messages)) diff --git a/webscout/Provider/OPENAI/venice.py b/webscout/Provider/OPENAI/venice.py index 87e3d4b8..7c0d2d2a 100644 --- a/webscout/Provider/OPENAI/venice.py +++ b/webscout/Provider/OPENAI/venice.py @@ -1,16 +1,21 @@ -import time -import uuid import json import random -from typing import List, Dict, Optional, Union, Generator, Any -from curl_cffi import CurlError +import time +import uuid +from typing import Any, Dict, Generator, List, Optional, Union + from curl_cffi.requests import Session # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, ) # Attempt to import LitAgent, fallback if not available @@ -45,7 +50,7 @@ def create( # Extract system message if present for systemPrompt parameter system_prompt = self._client.system_prompt filtered_messages = [] - + for msg in messages: if msg["role"] == "system": system_prompt = msg["content"] @@ -322,7 +327,7 @@ class Venice(OpenAICompatibleProvider): messages=[{"role": "user", "content": "Hello!"}] ) """ - required_auth = False + required_auth = False AVAILABLE_MODELS = [ "mistral-31-24b", "dolphin-3.0-mistral-24b", diff --git a/webscout/Provider/OPENAI/wisecat.py b/webscout/Provider/OPENAI/wisecat.py index f9304e1b..763bf0f5 100644 --- a/webscout/Provider/OPENAI/wisecat.py +++ b/webscout/Provider/OPENAI/wisecat.py @@ -1,16 +1,20 @@ +import re import time import uuid -import re -import json -from typing import List, Dict, Optional, Union, Generator, Any -from curl_cffi import CurlError +from typing import Any, Dict, Generator, List, Optional, Union + from curl_cffi.requests import Session # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, ) # Attempt to import LitAgent, fallback if not available @@ -334,11 +338,11 @@ def format_text(self, text: str) -> str: # Handle unicode escaping and quote unescaping text = text.encode().decode('unicode_escape') text = text.replace('\\\\', '\\').replace('\\"', '"') - + # Remove timing information text = re.sub(r'\(\d+\.?\d*s\)', '', text) text = re.sub(r'\(\d+\.?\d*ms\)', '', text) - + return text except Exception as e: # If any error occurs, return the original text diff --git a/webscout/Provider/OPENAI/writecream.py b/webscout/Provider/OPENAI/writecream.py index 075b5d05..dd7351e8 100644 --- a/webscout/Provider/OPENAI/writecream.py +++ b/webscout/Provider/OPENAI/writecream.py @@ -1,14 +1,20 @@ +import json import time import uuid +from typing import Any, Dict, Generator, List, Optional, Union + import requests -import json -from typing import List, Dict, Optional, Union, Generator, Any # Import base classes and utility structures -from .base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from .base import BaseChat, BaseCompletions, OpenAICompatibleProvider from .utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, ) # Attempt to import LitAgent, fallback if not available @@ -24,8 +30,8 @@ def __init__(self, client: 'Writecream'): def create( self, *, - model: str = None, # Not used by Writecream, for compatibility - messages: List[Dict[str, str]], + model: str, # Not used by Writecream, for compatibility + messages: List[Dict[str, Any]], max_tokens: Optional[int] = None, # Not used by Writecream stream: bool = False, temperature: Optional[float] = None, # Not used by Writecream @@ -158,9 +164,13 @@ def list(inner_self): if __name__ == "__main__": client = Writecream() response = client.chat.completions.create( + model="writecream", messages=[ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "What is the capital of France?"} ] ) - print(response.choices[0].message.content) + if isinstance(response, ChatCompletion): + print(response.choices[0].message.content) + else: + print(response) diff --git a/webscout/Provider/OPENAI/x0gpt.py b/webscout/Provider/OPENAI/x0gpt.py index 657a4448..57c35c34 100644 --- a/webscout/Provider/OPENAI/x0gpt.py +++ b/webscout/Provider/OPENAI/x0gpt.py @@ -1,17 +1,21 @@ +import re import time import uuid -import re -import json -from typing import List, Dict, Optional, Union, Generator, Any -from curl_cffi import CurlError -from curl_cffi.requests import Session +from typing import Any, Dict, Generator, List, Optional, Union + from curl_cffi.const import CurlHttpVersion +from curl_cffi.requests import Session # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, ) # Attempt to import LitAgent, fallback if not available @@ -388,4 +392,4 @@ def list(inner_self): {"role": "user", "content": "Hello! How are you today?"} ] ) - print(response.choices[0].message.content) \ No newline at end of file + print(response.choices[0].message.content) diff --git a/webscout/Provider/OPENAI/yep.py b/webscout/Provider/OPENAI/yep.py index b359688d..04b65ec9 100644 --- a/webscout/Provider/OPENAI/yep.py +++ b/webscout/Provider/OPENAI/yep.py @@ -1,14 +1,20 @@ +import json import time import uuid +from typing import Any, Dict, Generator, List, Optional, Union + import cloudscraper # Import cloudscraper -import json -from typing import List, Dict, Optional, Union, Generator, Any # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, get_system_prompt, count_tokens # Import count_tokens + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, # Import count_tokens + Choice, + ChoiceDelta, + CompletionUsage, + count_tokens, ) # Attempt to import LitAgent, fallback if not available @@ -194,7 +200,7 @@ def _create_stream( } yield chunk - except cloudscraper.exceptions.CloudflareChallengeError as e: + except cloudscraper.exceptions.CloudflareChallengeError: pass def _create_non_stream( @@ -391,4 +397,4 @@ def convert_model_name(self, model: str) -> str: print() # Add a newline at the end except Exception as e: - print(f"Streaming Test Failed: {e}") \ No newline at end of file + print(f"Streaming Test Failed: {e}") diff --git a/webscout/Provider/OPENAI/zenmux.py b/webscout/Provider/OPENAI/zenmux.py index a3f54625..d3c731cc 100644 --- a/webscout/Provider/OPENAI/zenmux.py +++ b/webscout/Provider/OPENAI/zenmux.py @@ -1,16 +1,17 @@ -import requests import json import time import uuid -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union + +import requests -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage, ) @@ -238,7 +239,7 @@ class Zenmux(OpenAICompatibleProvider): "z-ai/glm-4.6v-flash", ] - def __init__(self, browser: str = "chrome", api_key: str = None): + def __init__(self, browser: str = "chrome", api_key: Optional[str] = None): self.timeout = None self.base_url = "https://zenmux.ai/api/v1/chat/completions" self.session = requests.Session() @@ -281,12 +282,10 @@ def get_models(cls, api_key: Optional[str] = None): } try: from curl_cffi.requests import Session as CurlSession - from curl_cffi import CurlError curl_available = True except Exception: CurlSession = None - CurlError = Exception curl_available = False try: from webscout.litagent import LitAgent @@ -308,7 +307,7 @@ def get_models(cls, api_key: Optional[str] = None): pass if api_key: headers["Authorization"] = f"Bearer {api_key}" - if curl_available: + if curl_available and CurlSession is not None: session = CurlSession() response = session.get( "https://zenmux.ai/api/v1/models", diff --git a/webscout/Provider/Openai.py b/webscout/Provider/Openai.py index 4e3a958f..96d57472 100644 --- a/webscout/Provider/Openai.py +++ b/webscout/Provider/Openai.py @@ -1,17 +1,15 @@ import json -import os -from typing import Any, Dict, Optional, Generator, Union, List +from typing import Any, Dict, Generator, Optional, Union -from curl_cffi.requests import Session from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent + class OPENAI(Provider): """ A class to interact with the OpenAI API with LitAgent user-agent. @@ -19,12 +17,12 @@ class OPENAI(Provider): required_auth = True @classmethod - def get_models(cls, api_key: str = None): + def get_models(cls, api_key: Optional[str] = None): """Fetch available models from OpenAI API. - + Args: api_key (str, optional): OpenAI API key - + Returns: list: List of available model IDs """ @@ -36,21 +34,21 @@ def get_models(cls, api_key: str = None): headers = { "Authorization": f"Bearer {api_key}", } - + response = temp_session.get( "https://api.openai.com/v1/models", headers=headers, impersonate="chrome110" ) - + if response.status_code != 200: raise Exception(f"API request failed with status {response.status_code}: {response.text}") - + data = response.json() if "data" in data and isinstance(data["data"], list): return [model["id"] for model in data["data"] if "id" in model] raise Exception("Invalid response format from API") - + except (CurlError, Exception) as e: raise Exception(f"Failed to fetch models: {str(e)}") @@ -72,12 +70,12 @@ def __init__( top_p: float = 1, model: str = "gpt-3.5-turbo", timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, base_url: str = "https://api.openai.com/v1/chat/completions", system_prompt: str = "You are a helpful assistant.", browser: str = "chrome" @@ -167,9 +165,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: if optimizer in self.__available_optimizers: @@ -279,8 +278,9 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, + **kwargs: Any, ) -> Union[str, Generator[str, None, None]]: def for_stream_chat(): gen = self.ask( @@ -301,4 +301,4 @@ def for_non_stream_chat(): def get_message(self, response: dict) -> str: assert isinstance(response, dict), "Response should be of dict data-type only" - return response["text"] \ No newline at end of file + return response["text"] diff --git a/webscout/Provider/PI.py b/webscout/Provider/PI.py index ec7b2768..47e91bce 100644 --- a/webscout/Provider/PI.py +++ b/webscout/Provider/PI.py @@ -1,16 +1,21 @@ -from uuid import uuid4 -from curl_cffi.requests import Session -from curl_cffi import CurlError -import json import re import threading -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation, sanitize_stream # Import sanitize_stream -from webscout.AIutel import AwesomePrompts -from webscout.AIbase import Provider -from typing import Dict, Union, Any, Optional -from webscout.litagent import LitAgent +from typing import Any, Dict, Generator, Optional, Union +from uuid import uuid4 + +from curl_cffi import CurlError +from curl_cffi.requests import Session + from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) +from webscout.litagent import LitAgent + class PiAI(Provider): """ @@ -39,12 +44,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 2048, # Note: max_tokens is not used by this API timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, voice: bool = False, voice_name: str = "voice3", output_file: str = "PiAI.mp3", @@ -168,12 +173,13 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - voice: bool = None, - voice_name: str = None, - output_file: str = None - ) -> Union[dict, str, Any]: + voice: Optional[bool] = None, + voice_name: Optional[str] = None, + output_file: Optional[str] = None, + **kwargs: Any, + ) -> Response: """ Interact with Pi.ai by sending a prompt and receiving a response. Now supports raw streaming and non-streaming output, matching the pattern in other providers. @@ -292,13 +298,14 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - voice: bool = None, - voice_name: str = None, - output_file: str = None, + voice: Optional[bool] = None, + voice_name: Optional[str] = None, + output_file: Optional[str] = None, raw: bool = False, # Added raw parameter - ) -> Union[str, Any]: + **kwargs: Any, + ) -> Union[str, Generator[str, None, None]]: """ Generates a response based on the provided prompt. @@ -353,10 +360,11 @@ def stream_generator(): else: return self.get_message(response_data) - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """Retrieves message only from response""" - assert isinstance(response, dict), "Response should be of dict data-type only" - return response["text"] + if not isinstance(response, dict): + return str(response) + return response.get("text", "") def download_audio_threaded(self, voice_name: str, second_sid: str, output_file: str) -> None: """Downloads audio in a separate thread.""" @@ -396,9 +404,11 @@ def download_audio_threaded(self, voice_name: str, second_sid: str, output_file: ai = PiAI(timeout=60) print("[bold blue]Testing Chat (Stream):[/bold blue]") response = ai.chat("hi", stream=True, raw=False) - full_response = "" - for chunk in response: - print(chunk, end="", flush=True) + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end="", flush=True) + else: + print(response) except exceptions.FailedToGenerateResponseError as e: print(f"\n[bold red]API Error:[/bold red] {e}") except Exception as e: diff --git a/webscout/Provider/QwenLM.py b/webscout/Provider/QwenLM.py index 9f48ff9e..7d9c5225 100644 --- a/webscout/Provider/QwenLM.py +++ b/webscout/Provider/QwenLM.py @@ -5,7 +5,7 @@ from curl_cffi import Session from webscout import exceptions -from webscout.AIbase import Provider +from webscout.AIbase import Provider, Response from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream @@ -143,7 +143,8 @@ def ask( raw: bool = False, optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator[Any, None, None]]: + **kwargs: Any, + ) -> Response: """Chat with AI.""" conversation_prompt = self.conversation.gen_complete_prompt(prompt) @@ -307,9 +308,10 @@ def for_non_stream() -> str: return for_stream() if stream else for_non_stream() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """Extracts the message content from a response dict.""" - assert isinstance(response, dict), "Response should be a dict" + if not isinstance(response, dict): + return str(response) return response.get("text", "") if __name__ == "__main__": diff --git a/webscout/Provider/STT/__init__.py b/webscout/Provider/STT/__init__.py index c509c78a..a6284e69 100644 --- a/webscout/Provider/STT/__init__.py +++ b/webscout/Provider/STT/__init__.py @@ -3,12 +3,12 @@ # Base classes from webscout.Provider.STT.base import ( - STTCompatibleProvider, - BaseSTTTranscriptions, BaseSTTAudio, BaseSTTChat, - TranscriptionResponse, + BaseSTTTranscriptions, + STTCompatibleProvider, STTModels, + TranscriptionResponse, ) # Provider implementations diff --git a/webscout/Provider/STT/base.py b/webscout/Provider/STT/base.py index 5c245e42..8662d18d 100644 --- a/webscout/Provider/STT/base.py +++ b/webscout/Provider/STT/base.py @@ -8,41 +8,46 @@ import json import time from abc import ABC, abstractmethod -from typing import Any, Dict, Generator, List, Optional, Union, BinaryIO from pathlib import Path +from typing import Any, BinaryIO, Dict, Generator, List, Optional, Union # Import OpenAI response types from the main OPENAI module try: from webscout.Provider.OPENAI.pydantic_imports import ( - ChatCompletion, ChatCompletionChunk, Choice, ChoiceDelta, - Message, Usage, count_tokens + ChatCompletion, + ChatCompletionChunk, + Choice, + ChoiceDelta, + Message, + Usage, + count_tokens, ) except ImportError: # Fallback if pydantic_imports is not available from dataclasses import dataclass - + @dataclass class Usage: prompt_tokens: int = 0 completion_tokens: int = 0 total_tokens: int = 0 - + @dataclass class Message: role: str content: str - + @dataclass class Choice: index: int message: Message finish_reason: Optional[str] = None - + @dataclass class ChoiceDelta: content: Optional[str] = None role: Optional[str] = None - + @dataclass class ChatCompletionChunk: id: str @@ -50,7 +55,7 @@ class ChatCompletionChunk: created: int model: str object: str = "chat.completion.chunk" - + @dataclass class ChatCompletion: id: str @@ -59,43 +64,43 @@ class ChatCompletion: model: str usage: Usage object: str = "chat.completion" - + def count_tokens(text: str) -> int: return len(text.split()) class TranscriptionResponse: """Response object that mimics OpenAI's transcription response.""" - + def __init__(self, data: Dict[str, Any], response_format: str = "json"): self._data = data self._response_format = response_format - + @property def text(self) -> str: """Get the transcribed text.""" return self._data.get("text", "") - + @property def language(self) -> Optional[str]: """Get the detected language.""" return self._data.get("language") - + @property def duration(self) -> Optional[float]: """Get the audio duration.""" return self._data.get("duration") - + @property def segments(self) -> Optional[list]: """Get the segments with timestamps.""" return self._data.get("segments") - + @property def words(self) -> Optional[list]: """Get the words with timestamps.""" return self._data.get("words") - + def __str__(self) -> str: """Return string representation based on response format.""" if self._response_format == "text": @@ -106,42 +111,42 @@ def __str__(self) -> str: return self._to_vtt() else: # json or verbose_json return json.dumps(self._data, indent=2) - + def _to_srt(self) -> str: """Convert to SRT subtitle format.""" if not self.segments: return "" - + srt_content = [] for i, segment in enumerate(self.segments, 1): start_time = self._format_time_srt(segment.get("start", 0)) end_time = self._format_time_srt(segment.get("end", 0)) text = segment.get("text", "").strip() - + srt_content.append(f"{i}") srt_content.append(f"{start_time} --> {end_time}") srt_content.append(text) srt_content.append("") - + return "\n".join(srt_content) - + def _to_vtt(self) -> str: """Convert to VTT subtitle format.""" if not self.segments: return "WEBVTT\n\n" - + vtt_content = ["WEBVTT", ""] for segment in self.segments: start_time = self._format_time_vtt(segment.get("start", 0)) end_time = self._format_time_vtt(segment.get("end", 0)) text = segment.get("text", "").strip() - + vtt_content.append(f"{start_time} --> {end_time}") vtt_content.append(text) vtt_content.append("") - + return "\n".join(vtt_content) - + def _format_time_srt(self, seconds: float) -> str: """Format time for SRT format (HH:MM:SS,mmm).""" hours = int(seconds // 3600) @@ -149,7 +154,7 @@ def _format_time_srt(self, seconds: float) -> str: secs = int(seconds % 60) millisecs = int((seconds % 1) * 1000) return f"{hours:02d}:{minutes:02d}:{secs:02d},{millisecs:03d}" - + def _format_time_vtt(self, seconds: float) -> str: """Format time for VTT format (HH:MM:SS.mmm).""" hours = int(seconds // 3600) @@ -161,10 +166,10 @@ def _format_time_vtt(self, seconds: float) -> str: class BaseSTTTranscriptions(ABC): """Base class for STT transcriptions interface.""" - + def __init__(self, client): self._client = client - + @abstractmethod def create( self, @@ -183,7 +188,7 @@ def create( ) -> Union[TranscriptionResponse, Generator[str, None, None]]: """ Create a transcription of the given audio file. - + Args: model: Model to use for transcription file: Audio file to transcribe @@ -196,7 +201,7 @@ def create( timeout: Request timeout proxies: Proxy configuration **kwargs: Additional parameters - + Returns: TranscriptionResponse or generator of SSE strings if streaming """ @@ -205,10 +210,10 @@ def create( class BaseSTTAudio(ABC): """Base class for STT audio interface.""" - + def __init__(self, client): self.transcriptions = self._create_transcriptions(client) - + @abstractmethod def _create_transcriptions(self, client) -> BaseSTTTranscriptions: """Create the transcriptions interface.""" @@ -228,14 +233,14 @@ class STTCompatibleProvider(ABC): Abstract Base Class for STT providers mimicking the OpenAI structure. Requires a nested 'audio.transcriptions' structure. """ - + audio: BaseSTTAudio - + @abstractmethod def __init__(self, **kwargs: Any): """Initialize the STT provider.""" pass - + @property @abstractmethod def models(self): @@ -247,10 +252,10 @@ def models(self): class STTModels: """Models interface for STT providers.""" - + def __init__(self, available_models: List[str]): self._available_models = available_models - + def list(self) -> List[Dict[str, Any]]: """List available models.""" return [ @@ -266,7 +271,7 @@ def list(self) -> List[Dict[str, Any]]: __all__ = [ 'TranscriptionResponse', - 'BaseSTTTranscriptions', + 'BaseSTTTranscriptions', 'BaseSTTAudio', 'BaseSTTChat', 'STTCompatibleProvider', diff --git a/webscout/Provider/STT/elevenlabs.py b/webscout/Provider/STT/elevenlabs.py index c5e28e52..b0423845 100644 --- a/webscout/Provider/STT/elevenlabs.py +++ b/webscout/Provider/STT/elevenlabs.py @@ -5,25 +5,25 @@ speech-to-text transcription service. """ -import json -import time -import uuid from pathlib import Path -from typing import Any, Dict, Generator, List, Optional, Union, BinaryIO +from typing import Any, BinaryIO, Generator, List, Optional, Union import requests -from webscout.litagent import LitAgent -from webscout import exceptions +from webscout import exceptions +from webscout.litagent import LitAgent from webscout.Provider.STT.base import ( - BaseSTTTranscriptions, BaseSTTAudio, STTCompatibleProvider, - STTModels, TranscriptionResponse + BaseSTTAudio, + BaseSTTTranscriptions, + STTCompatibleProvider, + STTModels, + TranscriptionResponse, ) class ElevenLabsTranscriptions(BaseSTTTranscriptions): """ElevenLabs transcriptions interface.""" - + def create( self, *, @@ -181,12 +181,12 @@ def _create_stream( for line in response.iter_lines(decode_unicode=True): if line: yield line - + class ElevenLabsAudio(BaseSTTAudio): """ElevenLabs audio interface.""" - + def _create_transcriptions(self, client) -> ElevenLabsTranscriptions: return ElevenLabsTranscriptions(client) @@ -194,7 +194,7 @@ def _create_transcriptions(self, client) -> ElevenLabsTranscriptions: class ElevenLabsSTT(STTCompatibleProvider): """ OpenAI-compatible client for ElevenLabs STT API. - + Usage: client = ElevenLabsSTT() audio_file = open("audio.mp3", "rb") @@ -205,11 +205,11 @@ class ElevenLabsSTT(STTCompatibleProvider): ) print(transcription.text) """ - + AVAILABLE_MODELS = [ "scribe_v1", ] - + def __init__( self, model_id: str = "scribe_v1", @@ -226,14 +226,14 @@ def __init__( self.diarize = diarize self.timeout = timeout self.proxies = proxies - + # API configuration self.api_url = "https://api.elevenlabs.io/v1/speech-to-text" - + # Initialize interfaces self.audio = ElevenLabsAudio(self) self._models = STTModels(self.AVAILABLE_MODELS) - + @property def models(self): """Get models interface.""" @@ -262,4 +262,4 @@ def models(self): stream=True ) for chunk in stream: - print(chunk.strip()) \ No newline at end of file + print(chunk.strip()) diff --git a/webscout/Provider/Sambanova.py b/webscout/Provider/Sambanova.py index b7e5b17a..4b5d246a 100644 --- a/webscout/Provider/Sambanova.py +++ b/webscout/Provider/Sambanova.py @@ -1,13 +1,15 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError import json -from typing import Union, Any, Dict, Generator, Optional, List +from typing import Any, Dict, Generator, List, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers, Conversation, AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent as Lit + class Sambanova(Provider): """ A class to interact with the Sambanova API. @@ -28,42 +30,42 @@ def update_available_models(self): """Update the available models list from Sambanova API.""" if not self.api_key: return - + try: temp_session = Session() headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}", } - + response = temp_session.get( "https://api.sambanova.ai/v1/models", headers=headers, impersonate="chrome120" ) - + if response.status_code == 200: data = response.json() if "data" in data and isinstance(data["data"], list): new_models = [model['id'] for model in data['data'] if 'id' in model] if new_models: self.AVAILABLE_MODELS = new_models - + except Exception: pass def __init__( self, - api_key: str = None, + api_key: Optional[str] = None, is_conversation: bool = True, max_tokens: int = 4096, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "Meta-Llama-3.1-8B-Instruct", system_prompt: str = "You are a helpful AI assistant.", ): @@ -74,7 +76,7 @@ def __init__( self.model = model self.system_prompt = system_prompt self.timeout = timeout - + # Update models list dynamically if API key is provided if api_key: self.update_available_models() @@ -90,7 +92,7 @@ def __init__( "Content-Type": "application/json", "User-Agent": Lit().random(), } - + # Update curl_cffi session headers and proxies self.session.headers.update(self.headers) self.session.proxies = proxies @@ -120,11 +122,12 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, tools: Optional[List[Dict[str, Any]]] = None, tool_choice: Optional[Dict[str, Any]] = None, - ) -> Union[Any, Generator[Any, None, None]]: + **kwargs: Any, + ) -> Response: """Chat with AI using the Sambanova API.""" conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: @@ -156,13 +159,13 @@ def for_stream(): try: # Use curl_cffi session post with impersonate response = self.session.post( - self.base_url, - json=payload, - stream=True, + self.base_url, + json=payload, + stream=True, timeout=self.timeout, impersonate="chrome120" ) - response.raise_for_status() + response.raise_for_status() streaming_text = "" processed_stream = sanitize_stream( @@ -242,18 +245,18 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, tools: Optional[List[Dict[str, Any]]] = None, tool_choice: Optional[Dict[str, Any]] = None, raw: bool = False, ) -> Union[str, Generator[str, None, None]]: """Generate response `str`""" - + def for_stream_chat(): # ask() yields dicts or strings when streaming gen = self.ask( - prompt, stream=True, raw=raw, + prompt, stream=True, raw=raw, optimizer=optimizer, conversationally=conversationally, tools=tools, tool_choice=tool_choice ) @@ -261,14 +264,14 @@ def for_stream_chat(): if raw: yield response_dict else: - yield self.get_message(response_dict) + yield self.get_message(response_dict) def for_non_stream_chat(): # ask() returns dict or str when not streaming response_data = self.ask( prompt, stream=False, - raw=raw, + raw=raw, optimizer=optimizer, conversationally=conversationally, tools=tools, @@ -276,11 +279,11 @@ def for_non_stream_chat(): ) if raw: return response_data - return self.get_message(response_data) + return self.get_message(response_data) return for_stream_chat() if stream else for_non_stream_chat() - def get_message(self, response: Any) -> str: + def get_message(self, response: Response) -> str: """ Retrieves a clean message from the provided response. @@ -297,12 +300,15 @@ def get_message(self, response: Any) -> str: return response["text"] elif "tool_calls" in response: return json.dumps(response["tool_calls"]) - return "" + return str(response) if __name__ == "__main__": # Ensure curl_cffi is installed from rich import print ai = Sambanova(api_key='') response = ai.chat(input(">>> "), stream=True) - for chunk in response: - print(chunk, end="", flush=True) \ No newline at end of file + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end="", flush=True) + else: + print(response) diff --git a/webscout/Provider/TTI/__init__.py b/webscout/Provider/TTI/__init__.py index 0b763b6e..acbeddce 100644 --- a/webscout/Provider/TTI/__init__.py +++ b/webscout/Provider/TTI/__init__.py @@ -3,23 +3,23 @@ # Base classes from webscout.Provider.TTI.base import ( - TTICompatibleProvider, BaseImages, -) - -# Utility classes -from webscout.Provider.TTI.utils import ( - ImageData, - ImageResponse, + TTICompatibleProvider, ) # Provider implementations from webscout.Provider.TTI.claudeonline import ClaudeOnlineTTI from webscout.Provider.TTI.magicstudio import MagicStudioAI +from webscout.Provider.TTI.miragic import MiragicAI from webscout.Provider.TTI.pollinations import PollinationsAI from webscout.Provider.TTI.together import TogetherImage + +# Utility classes +from webscout.Provider.TTI.utils import ( + ImageData, + ImageResponse, +) from webscout.Provider.TTI.venice import VeniceAI -from webscout.Provider.TTI.miragic import MiragicAI # List of all exported names __all__ = [ @@ -37,4 +37,4 @@ "VeniceAI", "MiragicAI", -] \ No newline at end of file +] diff --git a/webscout/Provider/TTI/base.py b/webscout/Provider/TTI/base.py index 231dba36..e10bc2b6 100644 --- a/webscout/Provider/TTI/base.py +++ b/webscout/Provider/TTI/base.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import Any, Optional -from .utils import ImageResponse +from .utils import ImageResponse class BaseImages(ABC): diff --git a/webscout/Provider/TTI/magicstudio.py b/webscout/Provider/TTI/magicstudio.py index a96a8505..4706e8f4 100644 --- a/webscout/Provider/TTI/magicstudio.py +++ b/webscout/Provider/TTI/magicstudio.py @@ -1,16 +1,15 @@ -import requests import os -import uuid -import time import tempfile -from typing import Optional, List -from webscout.Provider.TTI.utils import ( - ImageData, - ImageResponse -) -from webscout.Provider.TTI.base import TTICompatibleProvider, BaseImages +import time +import uuid from io import BytesIO +from typing import Optional + +import requests + from webscout.litagent import LitAgent +from webscout.Provider.TTI.base import BaseImages, TTICompatibleProvider +from webscout.Provider.TTI.utils import ImageData, ImageResponse try: from PIL import Image diff --git a/webscout/Provider/TTI/miragic.py b/webscout/Provider/TTI/miragic.py index 50dc48ea..b14b904a 100644 --- a/webscout/Provider/TTI/miragic.py +++ b/webscout/Provider/TTI/miragic.py @@ -1,11 +1,13 @@ -import requests import json import random -import time -from typing import Optional, List, Any, Dict -from webscout.Provider.TTI.base import TTICompatibleProvider, BaseImages -from webscout.Provider.TTI.utils import ImageData, ImageResponse +from typing import Any, Optional + +import requests + from webscout.litagent import LitAgent +from webscout.Provider.TTI.base import BaseImages, TTICompatibleProvider +from webscout.Provider.TTI.utils import ImageData, ImageResponse + class Images(BaseImages): """Handles image generation requests for the Miragic AI provider.""" @@ -63,19 +65,19 @@ def create( width, height = int(parts[0]), int(parts[1]) except ValueError: pass - + width = max(256, min(2048, width)) height = max(256, min(2048, height)) - + enhance = kwargs.get("enhance_prompt", False) safe = kwargs.get("safe_filter", False) image_url = kwargs.get("image_url") - + images_data = [] - + for _ in range(n): current_seed = seed if seed is not None else random.randint(0, 2**32 - 1) - + payload = { "data": [ prompt, @@ -88,26 +90,26 @@ def create( safe ] } - + try: post_url = f"{self._client.api_endpoint}/call/generate_image_via_api_secure" resp = self._client.session.post(post_url, json=payload, timeout=timeout) resp.raise_for_status() - + event_id = resp.json().get("event_id") if not event_id: raise RuntimeError(f"Failed to obtain event_id: {resp.text}") - + stream_url = f"{self._client.api_endpoint}/call/generate_image_via_api_secure/{event_id}" image_url_result = None - + with self._client.session.get(stream_url, stream=True, timeout=timeout + 60) as stream_resp: stream_resp.raise_for_status() for line in stream_resp.iter_lines(): if not line: continue line_text = line.decode('utf-8') - + if line_text.startswith('data: '): data_str = line_text[6:] try: @@ -120,7 +122,7 @@ def create( continue except Exception as e: raise RuntimeError(f"Image generation failed: {e}") - + if image_url_result: images_data.append(ImageData(url=image_url_result)) else: @@ -131,15 +133,15 @@ def create( class MiragicAI(TTICompatibleProvider): """ Miragic AI TTI Provider implementation. - + Reverse engineered from the Hugging Face Space: https://huggingface.co/spaces/Miragic-AI/Miragic-AI-Image-Generator """ - + required_auth: bool = False working: bool = True AVAILABLE_MODELS = ["flux", "turbo", "gptimage"] - + def __init__(self, **kwargs: Any): """Initializes the MiragicAI provider with a persistent session.""" self.api_endpoint = "https://miragic-ai-miragic-ai-image-generator.hf.space/gradio_api" @@ -151,7 +153,7 @@ def __init__(self, **kwargs: Any): "Content-Type": "application/json" }) self.images = Images(self) - + @property def models(self): """Returns a list of available models for this provider.""" @@ -162,11 +164,11 @@ def list(inner_self): if __name__ == "__main__": from rich import print - + try: client = MiragicAI() print(f"Available Models: {client.models.list()}") - + print("Generating sample image...") response = client.images.create( prompt="A serene landscape with a lake and mountains, oil painting style", @@ -174,6 +176,6 @@ def list(inner_self): size="1024x1024" ) print(response) - + except Exception as error: - print(f"Error during execution: {error}") \ No newline at end of file + print(f"Error during execution: {error}") diff --git a/webscout/Provider/TTI/pollinations.py b/webscout/Provider/TTI/pollinations.py index 9bffc084..d0895c27 100644 --- a/webscout/Provider/TTI/pollinations.py +++ b/webscout/Provider/TTI/pollinations.py @@ -1,19 +1,17 @@ -import requests -from typing import Optional, List, Dict, Any, Union -from pathlib import Path -from requests.exceptions import RequestException -from webscout.Provider.TTI.utils import ( - ImageData, - ImageResponse -) -from webscout.Provider.TTI.base import TTICompatibleProvider, BaseImages -from io import BytesIO +import json import os +import random import tempfile -from webscout.litagent import LitAgent import time -import json -import random +from io import BytesIO +from typing import Optional + +import requests +from requests.exceptions import RequestException + +from webscout.litagent import LitAgent +from webscout.Provider.TTI.base import BaseImages, TTICompatibleProvider +from webscout.Provider.TTI.utils import ImageData, ImageResponse try: from PIL import Image diff --git a/webscout/Provider/TTI/together.py b/webscout/Provider/TTI/together.py index 21391620..c9584d5e 100644 --- a/webscout/Provider/TTI/together.py +++ b/webscout/Provider/TTI/together.py @@ -1,6 +1,6 @@ import json import random -from typing import Dict, Optional, Any, List +from typing import Dict, Optional import requests from requests.adapters import HTTPAdapter @@ -262,7 +262,7 @@ def get_models(cls, api_key: str = None): for model in models_data: if isinstance(model, dict) and model.get("type", "").lower() == "image": image_models.append(model["id"]) - + if image_models: return sorted(image_models) else: @@ -284,9 +284,9 @@ def update_available_models(cls, api_key=None): def __init__(self, api_key: str = None): """ Initialize the TogetherImage client. - + Args: - api_key (str, optional): Together.xyz API key. + api_key (str, optional): Together.xyz API key. """ self.api_key = api_key # Update available models if API key is provided @@ -331,4 +331,4 @@ def convert_model_name(self, model: str) -> str: response_format="url", timeout=120, ) - print(response) \ No newline at end of file + print(response) diff --git a/webscout/Provider/TTI/utils.py b/webscout/Provider/TTI/utils.py index b24d7a74..71494b3e 100644 --- a/webscout/Provider/TTI/utils.py +++ b/webscout/Provider/TTI/utils.py @@ -1,7 +1,9 @@ import time from typing import List, Optional + from pydantic import BaseModel, Field + class ImageData(BaseModel): url: Optional[str] = None b64_json: Optional[str] = None diff --git a/webscout/Provider/TTI/venice.py b/webscout/Provider/TTI/venice.py index ce4c5999..28b91fca 100644 --- a/webscout/Provider/TTI/venice.py +++ b/webscout/Provider/TTI/venice.py @@ -11,21 +11,20 @@ >>> print(response) """ -import requests -from typing import Optional, List, Dict, Any -from webscout.Provider.TTI.utils import ( - ImageData, - ImageResponse -) -from webscout.Provider.TTI.base import TTICompatibleProvider, BaseImages -from io import BytesIO -import os -import tempfile -from webscout.litagent import LitAgent -import time import json +import os import random import string +import tempfile +import time +from io import BytesIO +from typing import Optional + +import requests + +from webscout.litagent import LitAgent +from webscout.Provider.TTI.base import BaseImages, TTICompatibleProvider +from webscout.Provider.TTI.utils import ImageData, ImageResponse try: from PIL import Image @@ -57,7 +56,7 @@ def create( ) -> ImageResponse: """ Create images using Venice AI API. - + Args: model: The model to use for image generation prompt: Text description of the image to generate @@ -73,7 +72,7 @@ def create( cfg_scale: CFG scale for generation (default: 3.5) steps: Number of inference steps (default: 25) **kwargs: Additional parameters - + Returns: ImageResponse: The generated images """ @@ -172,7 +171,7 @@ def upload_file_alternative(img_bytes, image_format): request_id = ''.join(random.choices(string.ascii_letters + string.digits, k=7)) message_id = ''.join(random.choices(string.ascii_letters + string.digits, k=7)) user_id = f"user_anon_{''.join(random.choices(string.digits, k=10))}" - + # Generate seed if not provided if seed is None: seed = random.randint(0, 2**32 - 1) @@ -215,7 +214,7 @@ def upload_file_alternative(img_bytes, image_format): # Venice API returns binary image content directly img_bytes = resp.content - + # Convert to png or jpeg in memory with BytesIO(img_bytes) as input_io: with Image.open(input_io) as im: @@ -281,7 +280,7 @@ class VeniceAI(TTICompatibleProvider): "pony-realism-akash", "hidream", ] - + MODEL_NAMES = { "z-image-turbo": "Z-Image Turbo", "stable-diffusion-3.5-rev2": "Venice SD35", @@ -299,7 +298,7 @@ def __init__(self): self.api_endpoint = "https://outerface.venice.ai/api/inference/image" self.session = requests.Session() self.user_agent = LitAgent().random() - + # Set up headers based on the provided request details self.headers = { "accept": "*/*", @@ -320,7 +319,7 @@ def __init__(self): "x-venice-timestamp": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "x-venice-version": "interface@20251219.114735+e0ef642" } - + self.session.headers.update(self.headers) self.images = Images(self) @@ -328,23 +327,23 @@ def __init__(self): def models(self): """ Get available models for the Venice AI provider. - + Returns: Object with list() method that returns available models """ class _ModelList: def list(inner_self): return type(self).AVAILABLE_MODELS - + return _ModelList() if __name__ == "__main__": from rich import print - + # Example usage client = VeniceAI() - + try: response = client.images.create( model="stable-diffusion-3.5-rev2", @@ -357,4 +356,4 @@ def list(inner_self): print("Generated image successfully:") print(response) except Exception as e: - print(f"Error: {e}") \ No newline at end of file + print(f"Error: {e}") diff --git a/webscout/Provider/TTS/__init__.py b/webscout/Provider/TTS/__init__.py index 68ca3d69..47aa4687 100644 --- a/webscout/Provider/TTS/__init__.py +++ b/webscout/Provider/TTS/__init__.py @@ -3,13 +3,10 @@ # Base classes from webscout.Provider.TTS.base import ( - BaseTTSProvider, AsyncBaseTTSProvider, + BaseTTSProvider, ) -# Utility classes -from webscout.Provider.TTS.utils import SentenceTokenizer - # Provider implementations from webscout.Provider.TTS.deepgram import DeepgramTTS from webscout.Provider.TTS.elevenlabs import ElevenlabsTTS @@ -22,6 +19,9 @@ from webscout.Provider.TTS.speechma import SpeechMaTTS from webscout.Provider.TTS.streamElements import StreamElements +# Utility classes +from webscout.Provider.TTS.utils import SentenceTokenizer + # List of all exported names __all__ = [ # Base classes diff --git a/webscout/Provider/TTS/base.py b/webscout/Provider/TTS/base.py index 6a83c6fc..12564b89 100644 --- a/webscout/Provider/TTS/base.py +++ b/webscout/Provider/TTS/base.py @@ -4,31 +4,35 @@ import os import tempfile from pathlib import Path -from typing import Generator, Optional, Dict, List, Union +from typing import Generator + +from litprinter import ic + from webscout.AIbase import TTSProvider + class BaseTTSProvider(TTSProvider): """ Base class for TTS providers with OpenAI-compatible functionality. - + This class implements common methods and follows OpenAI TTS API patterns for speech generation, streaming, and audio handling. """ required_auth = False - + # Supported models (can be overridden by subclasses) SUPPORTED_MODELS = [ "gpt-4o-mini-tts", # Latest intelligent realtime model "tts-1", # Lower latency model "tts-1-hd" # Higher quality model ] - + # Supported voices (can be overridden by subclasses) SUPPORTED_VOICES = [ - "alloy", "ash", "ballad", "coral", "echo", + "alloy", "ash", "ballad", "coral", "echo", "fable", "nova", "onyx", "sage", "shimmer" ] - + # Supported output formats SUPPORTED_FORMATS = [ "mp3", # Default format @@ -38,62 +42,62 @@ class BaseTTSProvider(TTSProvider): "wav", # Uncompressed, low latency "pcm" # Raw samples, 24kHz 16-bit ] - + def __init__(self): """Initialize the base TTS provider.""" self.temp_dir = tempfile.mkdtemp(prefix="webscout_tts_") self.default_model = "gpt-4o-mini-tts" self.default_voice = "coral" self.default_format = "mp3" - + def validate_model(self, model: str) -> str: """ Validate and return the model name. - + Args: model (str): Model name to validate - + Returns: str: Validated model name - + Raises: ValueError: If model is not supported """ # If provider doesn't support models, return the model as-is if self.SUPPORTED_MODELS is None: return model - + if model not in self.SUPPORTED_MODELS: raise ValueError(f"Model '{model}' not supported. Available models: {', '.join(self.SUPPORTED_MODELS)}") return model - + def validate_voice(self, voice: str) -> str: """ Validate and return the voice name. - + Args: voice (str): Voice name to validate - + Returns: str: Validated voice name - + Raises: ValueError: If voice is not supported """ if voice not in self.SUPPORTED_VOICES: raise ValueError(f"Voice '{voice}' not supported. Available voices: {', '.join(self.SUPPORTED_VOICES)}") return voice - + def validate_format(self, response_format: str) -> str: """ Validate and return the response format. - + Args: response_format (str): Response format to validate - + Returns: str: Validated response format - + Raises: ValueError: If format is not supported """ @@ -104,54 +108,55 @@ def validate_format(self, response_format: str) -> str: def save_audio(self, audio_file: str, destination: str = None, verbose: bool = False) -> str: """ Save audio to a specific destination. - + Args: audio_file (str): Path to the source audio file destination (str, optional): Destination path. Defaults to current directory with timestamp. verbose (bool, optional): Whether to print debug information. Defaults to False. - + Returns: str: Path to the saved audio file - + Raises: FileNotFoundError: If the audio file doesn't exist """ import shutil import time - + source_path = Path(audio_file) - + if not source_path.exists(): raise FileNotFoundError(f"Audio file not found: {audio_file}") - + if destination is None: # Create a default destination with timestamp in current directory timestamp = int(time.time()) destination = os.path.join(os.getcwd(), f"speech_{timestamp}{source_path.suffix}") - + # Ensure the destination directory exists os.makedirs(os.path.dirname(os.path.abspath(destination)), exist_ok=True) - + # Copy the file shutil.copy2(source_path, destination) - + if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Audio saved to {destination}") - + ic.configureOutput(prefix='DEBUG| ') + ic(f"Audio saved to {destination}") + return destination - + def create_speech( - self, - input_text: str, + self, + input_text: str, model: str = None, - voice: str = None, + voice: str = None, response_format: str = None, instructions: str = None, verbose: bool = False ) -> str: """ Create speech from input text (OpenAI-compatible interface). - + Args: input_text (str): The text to convert to speech model (str, optional): The TTS model to use @@ -159,7 +164,7 @@ def create_speech( response_format (str, optional): Audio format (mp3, opus, aac, flac, wav, pcm) instructions (str, optional): Voice instructions for controlling speech aspects verbose (bool, optional): Whether to print debug information - + Returns: str: Path to the generated audio file """ @@ -167,12 +172,12 @@ def create_speech( model = model or self.default_model voice = voice or self.default_voice response_format = response_format or self.default_format - + # Validate parameters self.validate_model(model) self.validate_voice(voice) self.validate_format(response_format) - + # Call the provider-specific TTS implementation return self.tts( text=input_text, @@ -184,18 +189,18 @@ def create_speech( ) def stream_audio( - self, - input_text: str, + self, + input_text: str, model: str = None, - voice: str = None, + voice: str = None, response_format: str = None, instructions: str = None, - chunk_size: int = 1024, + chunk_size: int = 1024, verbose: bool = False ) -> Generator[bytes, None, None]: """ Stream audio in chunks with OpenAI-compatible parameters. - + Args: input_text (str): The text to convert to speech model (str, optional): The TTS model to use @@ -204,7 +209,7 @@ def stream_audio( instructions (str, optional): Voice instructions chunk_size (int, optional): Size of audio chunks to yield. Defaults to 1024. verbose (bool, optional): Whether to print debug information. Defaults to False. - + Yields: Generator[bytes, None, None]: Audio data chunks """ @@ -217,24 +222,24 @@ def stream_audio( instructions=instructions, verbose=verbose ) - + # Stream the file in chunks with open(audio_file, 'rb') as f: while chunk := f.read(chunk_size): yield chunk - + def tts(self, text: str, **kwargs) -> str: """ Abstract method for text-to-speech conversion. Must be implemented by subclasses. - + Args: text (str): The text to convert to speech **kwargs: Additional provider-specific parameters - + Returns: str: Path to the generated audio file - + Raises: NotImplementedError: If not implemented by subclass """ @@ -244,25 +249,25 @@ def tts(self, text: str, **kwargs) -> str: class AsyncBaseTTSProvider: """ Base class for async TTS providers with OpenAI-compatible functionality. - + This class implements common async methods following OpenAI TTS API patterns for speech generation, streaming, and audio handling. """ required_auth = False - + # Supported models (can be overridden by subclasses) SUPPORTED_MODELS = [ "gpt-4o-mini-tts", # Latest intelligent realtime model "tts-1", # Lower latency model "tts-1-hd" # Higher quality model ] - + # Supported voices (can be overridden by subclasses) SUPPORTED_VOICES = [ - "alloy", "ash", "ballad", "coral", "echo", + "alloy", "ash", "ballad", "coral", "echo", "fable", "nova", "onyx", "sage", "shimmer" ] - + # Supported output formats SUPPORTED_FORMATS = [ "mp3", # Default format @@ -272,62 +277,62 @@ class AsyncBaseTTSProvider: "wav", # Uncompressed, low latency "pcm" # Raw samples, 24kHz 16-bit ] - + def __init__(self): """Initialize the async base TTS provider.""" self.temp_dir = tempfile.mkdtemp(prefix="webscout_tts_") self.default_model = "gpt-4o-mini-tts" self.default_voice = "coral" self.default_format = "mp3" - + async def validate_model(self, model: str) -> str: """ Validate and return the model name. - + Args: model (str): Model name to validate - + Returns: str: Validated model name - + Raises: ValueError: If model is not supported """ # If provider doesn't support models, return the model as-is if self.SUPPORTED_MODELS is None: return model - + if model not in self.SUPPORTED_MODELS: raise ValueError(f"Model '{model}' not supported. Available models: {', '.join(self.SUPPORTED_MODELS)}") return model - + async def validate_voice(self, voice: str) -> str: """ Validate and return the voice name. - + Args: voice (str): Voice name to validate - + Returns: str: Validated voice name - + Raises: ValueError: If voice is not supported """ if voice not in self.SUPPORTED_VOICES: raise ValueError(f"Voice '{voice}' not supported. Available voices: {', '.join(self.SUPPORTED_VOICES)}") return voice - + async def validate_format(self, response_format: str) -> str: """ Validate and return the response format. - + Args: response_format (str): Response format to validate - + Returns: str: Validated response format - + Raises: ValueError: If format is not supported """ @@ -338,55 +343,56 @@ async def validate_format(self, response_format: str) -> str: async def save_audio(self, audio_file: str, destination: str = None, verbose: bool = False) -> str: """ Save audio to a specific destination asynchronously. - + Args: audio_file (str): Path to the source audio file destination (str, optional): Destination path. Defaults to current directory with timestamp. verbose (bool, optional): Whether to print debug information. Defaults to False. - + Returns: str: Path to the saved audio file - + Raises: FileNotFoundError: If the audio file doesn't exist """ + import asyncio import shutil import time - import asyncio - + source_path = Path(audio_file) - + if not source_path.exists(): raise FileNotFoundError(f"Audio file not found: {audio_file}") - + if destination is None: # Create a default destination with timestamp in current directory timestamp = int(time.time()) destination = os.path.join(os.getcwd(), f"speech_{timestamp}{source_path.suffix}") - + # Ensure the destination directory exists os.makedirs(os.path.dirname(os.path.abspath(destination)), exist_ok=True) - + # Copy the file using asyncio to avoid blocking await asyncio.to_thread(shutil.copy2, source_path, destination) - + if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Audio saved to {destination}") - + ic.configureOutput(prefix='DEBUG| ') + ic(f"Audio saved to {destination}") + return destination - + async def create_speech( - self, - input_text: str, + self, + input_text: str, model: str = None, - voice: str = None, + voice: str = None, response_format: str = None, instructions: str = None, verbose: bool = False ) -> str: """ Create speech from input text asynchronously (OpenAI-compatible interface). - + Args: input_text (str): The text to convert to speech model (str, optional): The TTS model to use @@ -394,7 +400,7 @@ async def create_speech( response_format (str, optional): Audio format (mp3, opus, aac, flac, wav, pcm) instructions (str, optional): Voice instructions for controlling speech aspects verbose (bool, optional): Whether to print debug information - + Returns: str: Path to the generated audio file """ @@ -402,12 +408,12 @@ async def create_speech( model = model or self.default_model voice = voice or self.default_voice response_format = response_format or self.default_format - + # Validate parameters await self.validate_model(model) await self.validate_voice(voice) await self.validate_format(response_format) - + # Call the provider-specific TTS implementation return await self.tts( text=input_text, @@ -417,20 +423,20 @@ async def create_speech( instructions=instructions, verbose=verbose ) - + async def stream_audio( - self, - input_text: str, + self, + input_text: str, model: str = None, - voice: str = None, + voice: str = None, response_format: str = None, instructions: str = None, - chunk_size: int = 1024, + chunk_size: int = 1024, verbose: bool = False ): """ Stream audio in chunks asynchronously with OpenAI-compatible parameters. - + Args: input_text (str): The text to convert to speech model (str, optional): The TTS model to use @@ -439,7 +445,7 @@ async def stream_audio( instructions (str, optional): Voice instructions chunk_size (int, optional): Size of audio chunks to yield. Defaults to 1024. verbose (bool, optional): Whether to print debug information. Defaults to False. - + Yields: AsyncGenerator[bytes, None]: Audio data chunks """ @@ -447,7 +453,7 @@ async def stream_audio( import aiofiles except ImportError: raise ImportError("The 'aiofiles' package is required for async streaming. Install it with 'pip install aiofiles'.") - + # Generate the audio file using create_speech audio_file = await self.create_speech( input_text=input_text, @@ -457,25 +463,25 @@ async def stream_audio( instructions=instructions, verbose=verbose ) - + # Stream the file in chunks async with aiofiles.open(audio_file, 'rb') as f: while chunk := await f.read(chunk_size): yield chunk - + async def tts(self, text: str, **kwargs) -> str: """ Abstract async method for text-to-speech conversion. Must be implemented by subclasses. - + Args: text (str): The text to convert to speech **kwargs: Additional provider-specific parameters - + Returns: str: Path to the generated audio file - + Raises: NotImplementedError: If not implemented by subclass """ - raise NotImplementedError("Subclasses must implement the async tts method") \ No newline at end of file + raise NotImplementedError("Subclasses must implement the async tts method") diff --git a/webscout/Provider/TTS/deepgram.py b/webscout/Provider/TTS/deepgram.py index 85ea52af..a0aa7d77 100644 --- a/webscout/Provider/TTS/deepgram.py +++ b/webscout/Provider/TTS/deepgram.py @@ -1,25 +1,24 @@ ################################################################################## ## Deepgram TTS Provider ## ################################################################################## -import time -import requests import pathlib -import base64 import tempfile -import json -from io import BytesIO -from webscout import exceptions +import time from concurrent.futures import ThreadPoolExecutor, as_completed -from webscout.litagent import LitAgent + +import requests from litprinter import ic +from webscout import exceptions +from webscout.litagent import LitAgent + try: from . import utils from .base import BaseTTSProvider except ImportError: # Handle direct execution - import sys import os + import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from webscout.Provider.TTS import utils from webscout.Provider.TTS.base import BaseTTSProvider @@ -27,7 +26,7 @@ class DeepgramTTS(BaseTTSProvider): """ Text-to-speech provider using the Deepgram Aura-2 API. - + This provider follows the OpenAI TTS API structure with support for: - Aura-2 next-gen voices - Low-latency real-time performance @@ -35,7 +34,7 @@ class DeepgramTTS(BaseTTSProvider): - Concurrent generation for long texts """ required_auth = False - + # Request headers headers: dict[str, str] = { "Accept": "*/*", @@ -45,18 +44,18 @@ class DeepgramTTS(BaseTTSProvider): "Referer": "https://deepgram.com/ai-voice-generator", "User-Agent": LitAgent().random() } - + # Supported Aura-2 voices SUPPORTED_MODELS = ["aura-2"] - + SUPPORTED_VOICES = [ - "thalia", "odysseus", "harmonia", "theia", "electra", + "thalia", "odysseus", "harmonia", "theia", "electra", "arcas", "amalthea", "helena", "hyperion", "apollo", "luna", # Legacy Aura-1 voices (if still supported by the endpoint) "asteria", "luna", "stella", "athena", "hera", "zeus", "orpheus", "arcas", "perseus", "angus", "orion", "helios" ] - + # Voice mapping for Deepgram API compatibility voice_mapping = { # Aura-2 @@ -87,7 +86,7 @@ class DeepgramTTS(BaseTTSProvider): def __init__(self, timeout: int = 30, proxies: dict = None): """ Initialize the Deepgram TTS client. - + Args: timeout (int): Request timeout in seconds proxies (dict): Proxy configuration @@ -102,12 +101,12 @@ def __init__(self, timeout: int = 30, proxies: dict = None): self.default_voice = "thalia" def tts( - self, - text: str, + self, + text: str, model: str = "aura-2", # Dummy model param for compatibility - voice: str = "thalia", + voice: str = "thalia", response_format: str = "mp3", - instructions: str = None, + instructions: str = None, verbose: bool = True ) -> str: """ @@ -124,10 +123,10 @@ def tts( """ if not text: raise ValueError("Input text must be a non-empty string") - + # Map voice to Deepgram API format voice_id = self.voice_mapping.get(voice.lower(), f"aura-2-{voice.lower()}-en") - + # Create temporary file file_extension = f".{response_format}" filename = pathlib.Path(tempfile.mktemp(suffix=file_extension, dir=self.temp_dir)) @@ -135,15 +134,17 @@ def tts( # Split text into sentences for long inputs sentences = utils.split_sentences(text) if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"DeepgramTTS: Processing {len(sentences)} chunks") - ic.configureOutput(prefix='DEBUG| '); ic(f"Voice: {voice} -> {voice_id}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"DeepgramTTS: Processing {len(sentences)} chunks") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Voice: {voice} -> {voice_id}") def generate_audio_for_chunk(part_text: str, part_number: int): max_retries = 3 for attempt in range(max_retries): try: payload = { - "text": part_text, + "text": part_text, "model": voice_id, "demoType": "voice-generator", "params": "tag=landingpage-aivoicegenerator" @@ -157,12 +158,14 @@ def generate_audio_for_chunk(part_text: str, part_number: int): if response.content: if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Chunk {part_number} processed successfully") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Chunk {part_number} processed successfully") return part_number, response.content except requests.RequestException as e: if verbose: - ic.configureOutput(prefix='WARNING| '); ic(f"Error processing chunk {part_number}: {e}. Retrying {attempt+1}/{max_retries}") + ic.configureOutput(prefix='WARNING| ') + ic(f"Error processing chunk {part_number}: {e}. Retrying {attempt+1}/{max_retries}") time.sleep(1) raise exceptions.FailedToGenerateResponseError(f"Failed to generate audio for chunk {part_number}") @@ -184,8 +187,9 @@ def generate_audio_for_chunk(part_text: str, part_number: int): f.write(audio_chunks[i]) if verbose: - ic.configureOutput(prefix='INFO| '); ic(f"Audio saved to {filename}") - + ic.configureOutput(prefix='INFO| ') + ic(f"Audio saved to {filename}") + return str(filename) except Exception as e: diff --git a/webscout/Provider/TTS/elevenlabs.py b/webscout/Provider/TTS/elevenlabs.py index 4c40b11e..dc30852d 100644 --- a/webscout/Provider/TTS/elevenlabs.py +++ b/webscout/Provider/TTS/elevenlabs.py @@ -2,23 +2,23 @@ ## ElevenLabs TTS Provider ## ################################################################################## import os -import requests import pathlib import tempfile -import time -from io import BytesIO -from webscout import exceptions -from webscout.litagent import LitAgent from concurrent.futures import ThreadPoolExecutor, as_completed + +import requests from litprinter import ic +from webscout import exceptions +from webscout.litagent import LitAgent + try: from . import utils from .base import BaseTTSProvider except ImportError: # Handle direct execution - import sys import os + import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from webscout.Provider.TTS import utils from webscout.Provider.TTS.base import BaseTTSProvider @@ -26,12 +26,12 @@ class ElevenlabsTTS(BaseTTSProvider): """ Text-to-speech provider using the ElevenLabs API. - - This provider supports both authenticated (with API key) and + + This provider supports both authenticated (with API key) and unauthenticated (limited) usage if available. """ required_auth = True - + # Supported models SUPPORTED_MODELS = [ "eleven_multilingual_v2", @@ -41,21 +41,21 @@ class ElevenlabsTTS(BaseTTSProvider): "eleven_turbo_v2", "eleven_monolingual_v1" ] - + # Request headers headers: dict[str, str] = { "User-Agent": LitAgent().random(), "Accept": "audio/mpeg", "Content-Type": "application/json", } - + # ElevenLabs voices SUPPORTED_VOICES = [ - "brian", "alice", "bill", "callum", "charlie", "charlotte", - "chris", "daniel", "eric", "george", "jessica", "laura", + "brian", "alice", "bill", "callum", "charlie", "charlotte", + "chris", "daniel", "eric", "george", "jessica", "laura", "liam", "lily", "matilda", "sarah", "will" ] - + # Voice mapping voice_mapping = { "brian": "nPczCjzI2devNBz1zQrb", @@ -80,7 +80,7 @@ class ElevenlabsTTS(BaseTTSProvider): def __init__(self, api_key: str = None, timeout: int = 30, proxies: dict = None): """ Initialize the ElevenLabs TTS client. - + Args: api_key (str): ElevenLabs API key. If None, tries to use unauthenticated endpoint. timeout (int): Request timeout in seconds. @@ -99,10 +99,10 @@ def __init__(self, api_key: str = None, timeout: int = 30, proxies: dict = None) self.default_voice = "brian" def tts( - self, - text: str, + self, + text: str, model: str = "eleven_multilingual_v2", - voice: str = "brian", + voice: str = "brian", response_format: str = "mp3", verbose: bool = True ) -> str: @@ -111,15 +111,16 @@ def tts( """ if not text: raise ValueError("Input text must be a non-empty string") - + voice_id = self.voice_mapping.get(voice.lower(), voice) - + file_extension = f".{response_format}" filename = pathlib.Path(tempfile.mktemp(suffix=file_extension, dir=self.temp_dir)) sentences = utils.split_sentences(text) if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"ElevenlabsTTS: Processing {len(sentences)} chunks") + ic.configureOutput(prefix='DEBUG| ') + ic(f"ElevenlabsTTS: Processing {len(sentences)} chunks") def generate_chunk(part_text: str, part_num: int): payload = { @@ -135,7 +136,7 @@ def generate_chunk(part_text: str, part_num: int): if not self.api_key: # Some public endpoints might still work without key but they are very restricted params['allow_unauthenticated'] = '1' - + response = self.session.post(url, json=payload, params=params, timeout=self.timeout) if response.status_code == 401 and not self.api_key: raise exceptions.FailedToGenerateResponseError("ElevenLabs requires an API key for this request.") @@ -149,13 +150,14 @@ def generate_chunk(part_text: str, part_num: int): for future in as_completed(futures): idx, data = future.result() audio_chunks[idx] = data - + with open(filename, 'wb') as f: for i in sorted(audio_chunks.keys()): f.write(audio_chunks[i]) - + if verbose: - ic.configureOutput(prefix='INFO| '); ic(f"Audio saved to {filename}") + ic.configureOutput(prefix='INFO| ') + ic(f"Audio saved to {filename}") return str(filename) except Exception as e: raise exceptions.FailedToGenerateResponseError(f"ElevenLabs TTS failed: {e}") diff --git a/webscout/Provider/TTS/freetts.py b/webscout/Provider/TTS/freetts.py index cf7e9344..48aea4cd 100644 --- a/webscout/Provider/TTS/freetts.py +++ b/webscout/Provider/TTS/freetts.py @@ -1,28 +1,29 @@ ################################################################################## ## FreeTTS Provider ## ################################################################################## -import os -import requests -import time import pathlib import tempfile -from datetime import datetime -from webscout.Provider.TTS.base import BaseTTSProvider -from webscout.litagent import LitAgent +import time + +import requests from litprinter import ic +from webscout import exceptions +from webscout.litagent import LitAgent +from webscout.Provider.TTS.base import BaseTTSProvider + class FreeTTS(BaseTTSProvider): """ Text-to-speech provider using the FreeTTS.ru API. - + Features: - Multiple languages (Russian, English, Ukrainian, etc.) - High-quality neural voices - Supports long texts via polling """ required_auth = False - + headers = { "Accept": "*/*", "Accept-Language": "en-US,en;q=0.9", @@ -31,14 +32,14 @@ class FreeTTS(BaseTTSProvider): "Referer": "https://freetts.ru/", "User-Agent": LitAgent().random() } - + # Supported formats SUPPORTED_FORMATS = ["mp3"] def __init__(self, lang="ru-RU", timeout: int = 30, proxies: dict = None): """ Initialize the FreeTTS TTS client. - + Args: lang (str): Language code for voice filtering timeout (int): Request timeout in seconds @@ -50,13 +51,13 @@ def __init__(self, lang="ru-RU", timeout: int = 30, proxies: dict = None): self.api_url = f"{self.base_url}/api/synthesis" self.list_url = f"{self.base_url}/api/list" self.history_url = f"{self.base_url}/api/history" - + self.session = requests.Session() self.session.headers.update(self.headers) if proxies: self.session.proxies.update(proxies) self.timeout = timeout - + self.voices = {} self.load_voices() # Set default voice to first available for requested lang @@ -83,7 +84,8 @@ def load_voices(self): if v_id not in self.SUPPORTED_VOICES: self.SUPPORTED_VOICES.append(v_id) except Exception as e: - ic.configureOutput(prefix='WARNING| '); ic(f"Error loading FreeTTS voices: {e}") + ic.configureOutput(prefix='WARNING| ') + ic(f"Error loading FreeTTS voices: {e}") def _get_default_voice(self): for v_id, info in self.voices.items(): @@ -96,12 +98,12 @@ def get_available_voices(self): return [v_id for v_id, info in self.voices.items() if info["lang"] == self.lang] def tts( - self, - text: str, + self, + text: str, model: str = None, # Dummy for compatibility - voice: str = None, + voice: str = None, response_format: str = "mp3", - instructions: str = None, + instructions: str = None, verbose: bool = True ) -> str: """ @@ -109,7 +111,7 @@ def tts( """ if not text: raise ValueError("Input text must be a non-empty string") - + voice = voice or self.default_voice if not voice: raise ValueError("No voices available") @@ -123,28 +125,31 @@ def tts( try: # Step 1: Start synthesis if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"FreeTTS: Starting synthesis for voice {voice}") - + ic.configureOutput(prefix='DEBUG| ') + ic(f"FreeTTS: Starting synthesis for voice {voice}") + response = self.session.post(self.api_url, json=payload, timeout=self.timeout) response.raise_for_status() - + # Step 2: Poll for completion max_polls = 20 poll_interval = 2 - + for i in range(max_polls): poll_resp = self.session.get(self.api_url, timeout=self.timeout) poll_resp.raise_for_status() poll_data = poll_resp.json() - + if poll_data.get("status") == 200 or poll_data.get("message") == "Обработка: 100%": if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"FreeTTS: Synthesis completed") + ic.configureOutput(prefix='DEBUG| ') + ic("FreeTTS: Synthesis completed") break - + if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"FreeTTS: {poll_data.get('message', 'Processing...')}") - + ic.configureOutput(prefix='DEBUG| ') + ic(f"FreeTTS: {poll_data.get('message', 'Processing...')}") + time.sleep(poll_interval) else: raise exceptions.FailedToGenerateResponseError("FreeTTS synthesis timed out") @@ -153,7 +158,7 @@ def tts( hist_resp = self.session.get(self.history_url, timeout=self.timeout) hist_resp.raise_for_status() hist_data = hist_resp.json() - + if hist_data.get("status") == "success": history = hist_data.get("data", []) if history: @@ -163,18 +168,19 @@ def tts( if audio_url: if not audio_url.startswith("http"): audio_url = self.base_url + audio_url - + # Download the file audio_file_resp = self.session.get(audio_url, timeout=self.timeout) audio_file_resp.raise_for_status() - + # Save to temp file filename = pathlib.Path(tempfile.mktemp(suffix=f".{response_format}", dir=self.temp_dir)) with open(filename, "wb") as f: f.write(audio_file_resp.content) - + if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"FreeTTS: Audio saved to {filename}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"FreeTTS: Audio saved to {filename}") return str(filename) raise exceptions.FailedToGenerateResponseError("Failed to retrieve audio URL from history") diff --git a/webscout/Provider/TTS/murfai.py b/webscout/Provider/TTS/murfai.py index 45dd341a..3f88d6a0 100644 --- a/webscout/Provider/TTS/murfai.py +++ b/webscout/Provider/TTS/murfai.py @@ -1,12 +1,16 @@ -import time -import requests import pathlib import tempfile +import time +from concurrent.futures import ThreadPoolExecutor, as_completed from io import BytesIO from urllib.parse import urlencode + +import requests +from litprinter import ic + from webscout import exceptions from webscout.litagent import LitAgent -from concurrent.futures import ThreadPoolExecutor, as_completed + from . import utils from .base import BaseTTSProvider @@ -14,7 +18,7 @@ class MurfAITTS(BaseTTSProvider): """ Text-to-speech provider using the MurfAITTS API with OpenAI-compatible interface. - + This provider follows the OpenAI TTS API structure with support for: - Multiple TTS models (gpt-4o-mini-tts, tts-1, tts-1-hd) - Multiple voices with OpenAI-style naming @@ -23,26 +27,26 @@ class MurfAITTS(BaseTTSProvider): - Streaming support """ required_auth = False - + # Override supported models for MurfAI (set to None as requested) SUPPORTED_MODELS = None - + # Override supported voices with real MurfAI voice names SUPPORTED_VOICES = [ "Hazel", "Marcus", "Samantha", "Natalie", "Michelle", "Ken", "Clint", "Amit", "Priya" ] - + # Override supported formats SUPPORTED_FORMATS = [ "mp3", # Default format for MurfAI "wav" # Alternative format ] - + # Request headers headers: dict[str, str] = { "User-Agent": LitAgent().random() } - + # Voice mapping from real names to MurfAI voice IDs voice_mapping: dict[str, str] = { "Hazel": "en-UK-hazel", @@ -68,13 +72,13 @@ def __init__(self, timeout: int = 20, proxies: dict = None): def tts(self, text: str, voice: str = "Hazel", verbose: bool = False, **kwargs) -> str: """ Converts text to speech using the MurfAITTS API and saves it to a file. - + Args: text (str): The text to convert to speech voice (str): The voice to use (default: "Hazel") verbose (bool): Whether to print debug information **kwargs: Additional parameters (model, response_format, instructions) - + Returns: str: Path to the generated audio file """ @@ -83,21 +87,21 @@ def tts(self, text: str, voice: str = "Hazel", verbose: bool = False, **kwargs) raise ValueError("Input text must be a non-empty string") if len(text) > 10000: raise ValueError("Input text exceeds maximum allowed length of 10,000 characters") - + # Use default voice if not provided if voice is None: voice = "Hazel" - + # Validate voice using base class method self.validate_voice(voice) - + # Map real voice name to MurfAI voice ID voice_id = self.voice_mapping.get(voice, "en-UK-hazel") # Default to Hazel - + # Get response format from kwargs or use default response_format = kwargs.get('response_format', 'mp3') response_format = self.validate_format(response_format) - + # Create temporary file with appropriate extension file_extension = f".{response_format}" if response_format != "pcm" else ".wav" filename = pathlib.Path(tempfile.mktemp(suffix=file_extension, dir=self.temp_dir)) @@ -120,14 +124,17 @@ def generate_audio_for_chunk(part_text: str, part_number: int): # Check if the request was successful if response.ok and response.status_code == 200: if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Chunk {part_number} processed successfully") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Chunk {part_number} processed successfully") return part_number, response.content else: if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"No data received for chunk {part_number}. Retrying...") + ic.configureOutput(prefix='DEBUG| ') + ic(f"No data received for chunk {part_number}. Retrying...") except requests.RequestException as e: if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Error for chunk {part_number}: {e}. Retrying...") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Error for chunk {part_number}: {e}. Retrying...") time.sleep(1) try: # Using ThreadPoolExecutor to handle requests concurrently @@ -145,25 +152,29 @@ def generate_audio_for_chunk(part_text: str, part_number: int): audio_chunks[part_number] = audio_data # Store the audio data in correct sequence except Exception as e: if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Failed to generate audio for chunk {chunk_num}: {e}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Failed to generate audio for chunk {chunk_num}: {e}") # Combine audio chunks in the correct sequence combined_audio = BytesIO() for part_number in sorted(audio_chunks.keys()): combined_audio.write(audio_chunks[part_number]) if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Added chunk {part_number} to the combined file.") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Added chunk {part_number} to the combined file.") # Save the combined audio data to a single file with open(filename, 'wb') as f: f.write(combined_audio.getvalue()) if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Final Audio Saved as {filename}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Final Audio Saved as {filename}") return filename.as_posix() except requests.exceptions.RequestException as e: if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Failed to perform the operation: {e}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Failed to perform the operation: {e}") raise exceptions.FailedToGenerateResponseError( f"Failed to perform the operation: {e}" ) @@ -171,7 +182,7 @@ def generate_audio_for_chunk(part_text: str, part_number: int): def create_speech( self, input: str, - model: str = "gpt-4o-mini-tts", + model: str = "gpt-4o-mini-tts", voice: str = "Hazel", response_format: str = "mp3", instructions: str = None, @@ -179,7 +190,7 @@ def create_speech( ) -> str: """ OpenAI-compatible speech creation interface. - + Args: input (str): The text to convert to speech model (str): The TTS model to use @@ -187,7 +198,7 @@ def create_speech( response_format (str): Audio format instructions (str): Voice instructions (not used by MurfAI) verbose (bool): Whether to print debug information - + Returns: str: Path to the generated audio file """ @@ -210,7 +221,7 @@ def stream_audio( ): """ Stream audio response in chunks. - + Args: input (str): The text to convert to speech model (str): The TTS model to use @@ -219,7 +230,7 @@ def stream_audio( instructions (str): Voice instructions chunk_size (int): Size of audio chunks to yield verbose (bool): Whether to print debug information - + Yields: bytes: Audio data chunks """ @@ -232,7 +243,7 @@ def stream_audio( instructions=instructions, verbose=verbose ) - + # Stream the file in chunks with open(audio_file, 'rb') as f: while chunk := f.read(chunk_size): @@ -243,7 +254,8 @@ def stream_audio( murfai = MurfAITTS() text = "This is a test of the MurfAITTS text-to-speech API. It supports multiple sentences and advanced logging." - ic.configureOutput(prefix='DEBUG| '); ic("Generating audio...") + ic.configureOutput(prefix='DEBUG| ') + ic("Generating audio...") try: audio_file = murfai.create_speech( input=text, @@ -252,6 +264,8 @@ def stream_audio( response_format="mp3", verbose=True ) - ic.configureOutput(prefix='INFO| '); ic(f"Audio saved to: {audio_file}") + ic.configureOutput(prefix='INFO| ') + ic(f"Audio saved to: {audio_file}") except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Error: {e}") \ No newline at end of file + ic.configureOutput(prefix='ERROR| ') + ic(f"Error: {e}") diff --git a/webscout/Provider/TTS/openai_fm.py b/webscout/Provider/TTS/openai_fm.py index fbe80142..a5fc4857 100644 --- a/webscout/Provider/TTS/openai_fm.py +++ b/webscout/Provider/TTS/openai_fm.py @@ -1,21 +1,21 @@ ################################################################################## ## OpenAI.fm TTS Provider ## ################################################################################## -import time -import requests import pathlib import tempfile -from io import BytesIO + +import requests +from litprinter import ic + from webscout import exceptions from webscout.litagent import LitAgent -from concurrent.futures import ThreadPoolExecutor, as_completed -from webscout.Provider.TTS import utils from webscout.Provider.TTS.base import BaseTTSProvider + class OpenAIFMTTS(BaseTTSProvider): """ Text-to-speech provider using the OpenAI.fm API with OpenAI-compatible interface. - + This provider follows the OpenAI TTS API structure with support for: - Multiple TTS models (gpt-4o-mini-tts, tts-1, tts-1-hd) - 11 built-in voices optimized for English @@ -24,7 +24,7 @@ class OpenAIFMTTS(BaseTTSProvider): - Streaming support """ required_auth = False - + # Request headers headers = { "accept": "*/*", @@ -37,14 +37,14 @@ class OpenAIFMTTS(BaseTTSProvider): "user-agent": LitAgent().random(), "referer": "https://www.openai.fm" } - + # Override supported models for OpenAI.fm SUPPORTED_MODELS = [ "gpt-4o-mini-tts", # Latest intelligent realtime model - "tts-1", # Lower latency model + "tts-1", # Lower latency model "tts-1-hd" # Higher quality model ] - + # OpenAI.fm supported voices (11 built-in voices) SUPPORTED_VOICES = [ "alloy", # Neutral voice with balanced tone @@ -58,11 +58,11 @@ class OpenAIFMTTS(BaseTTSProvider): "sage", # Measured and contemplative voice "shimmer" # Bright and optimistic voice ] - + # Voice mapping for API compatibility voice_mapping = { "alloy": "alloy", - "ash": "ash", + "ash": "ash", "ballad": "ballad", "coral": "coral", "echo": "echo", @@ -76,7 +76,7 @@ class OpenAIFMTTS(BaseTTSProvider): def __init__(self, timeout: int = 20, proxies: dict = None): """ Initialize the OpenAI.fm TTS client. - + Args: timeout (int): Request timeout in seconds proxies (dict): Proxy configuration @@ -90,12 +90,12 @@ def __init__(self, timeout: int = 20, proxies: dict = None): self.timeout = timeout def tts( - self, - text: str, + self, + text: str, model: str = "gpt-4o-mini-tts", - voice: str = "coral", + voice: str = "coral", response_format: str = "mp3", - instructions: str = None, + instructions: str = None, verbose: bool = True ) -> str: """ @@ -121,24 +121,24 @@ def tts( raise ValueError("Input text must be a non-empty string") if len(text) > 10000: raise ValueError("Input text exceeds maximum allowed length of 10,000 characters") - + # Validate model, voice, and format using base class methods model = self.validate_model(model) voice = self.validate_voice(voice) response_format = self.validate_format(response_format) - + # Map voice to API format voice_id = self.voice_mapping.get(voice, voice) - + # Set default instructions if not provided if instructions is None: instructions = "Speak in a cheerful and positive tone." - + # Create temporary file with appropriate extension file_extension = f".{response_format}" if response_format != "pcm" else ".wav" with tempfile.NamedTemporaryFile(suffix=file_extension, dir=self.temp_dir, delete=False) as temp_file: filename = pathlib.Path(temp_file.name) - + # Prepare parameters for the API request params = { "input": text, @@ -156,33 +156,40 @@ def tts( timeout=self.timeout ) response.raise_for_status() - + # Validate response content if not response.content: raise exceptions.FailedToGenerateResponseError("Empty response from API") - + # Save the audio file with open(filename, "wb") as f: f.write(response.content) - + if verbose: - ic.configureOutput(prefix='DEBUG| '); ic("Speech generated successfully") - ic.configureOutput(prefix='DEBUG| '); ic(f"Model: {model}") - ic.configureOutput(prefix='DEBUG| '); ic(f"Voice: {voice}") - ic.configureOutput(prefix='DEBUG| '); ic(f"Format: {response_format}") - ic.configureOutput(prefix='DEBUG| '); ic(f"Audio saved to {filename}") - + ic.configureOutput(prefix='DEBUG| ') + ic("Speech generated successfully") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Model: {model}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Voice: {voice}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Format: {response_format}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Audio saved to {filename}") + return filename.as_posix() - + except requests.exceptions.RequestException as e: if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Failed to generate speech: {e}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Failed to generate speech: {e}") raise exceptions.FailedToGenerateResponseError( f"Failed to generate speech: {e}" ) except Exception as e: if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Unexpected error: {e}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Unexpected error: {e}") raise exceptions.FailedToGenerateResponseError( f"Unexpected error during speech generation: {e}" ) @@ -190,7 +197,7 @@ def tts( def create_speech( self, input: str, - model: str = "gpt-4o-mini-tts", + model: str = "gpt-4o-mini-tts", voice: str = "coral", response_format: str = "mp3", instructions: str = None, @@ -198,7 +205,7 @@ def create_speech( ) -> str: """ OpenAI-compatible speech creation interface. - + Args: input (str): The text to convert to speech model (str): The TTS model to use @@ -206,7 +213,7 @@ def create_speech( response_format (str): Audio format instructions (str): Voice instructions verbose (bool): Whether to print debug information - + Returns: str: Path to the generated audio file """ @@ -222,7 +229,7 @@ def create_speech( def with_streaming_response(self): """ Return a streaming response context manager (OpenAI-compatible). - + Returns: StreamingResponseContextManager: Context manager for streaming responses """ @@ -233,29 +240,29 @@ class StreamingResponseContextManager: """ Context manager for streaming TTS responses (OpenAI-compatible). """ - + def __init__(self, tts_provider: OpenAIFMTTS): self.tts_provider = tts_provider self.audio_file = None - + def create( self, input: str, model: str = "gpt-4o-mini-tts", - voice: str = "coral", + voice: str = "coral", response_format: str = "mp3", instructions: str = None ): """ Create speech with streaming capability. - + Args: input (str): The text to convert to speech model (str): The TTS model to use voice (str): The voice to use response_format (str): Audio format instructions (str): Voice instructions - + Returns: StreamingResponse: Streaming response object """ @@ -267,10 +274,10 @@ def create( instructions=instructions ) return StreamingResponse(self.audio_file) - + def __enter__(self): return self - + def __exit__(self, exc_type, exc_val, exc_tb): pass @@ -279,36 +286,36 @@ class StreamingResponse: """ Streaming response object for TTS audio (OpenAI-compatible). """ - + def __init__(self, audio_file: str): self.audio_file = audio_file - + def __enter__(self): """Enter the context manager.""" return self - + def __exit__(self, exc_type, exc_val, exc_tb): """Exit the context manager.""" pass - + def stream_to_file(self, file_path: str, chunk_size: int = 1024): """ Stream audio content to a file. - + Args: file_path (str): Destination file path chunk_size (int): Size of chunks to read/write """ import shutil shutil.copy2(self.audio_file, file_path) - + def iter_bytes(self, chunk_size: int = 1024): """ Iterate over audio bytes in chunks. - + Args: chunk_size (int): Size of chunks to yield - + Yields: bytes: Audio data chunks """ @@ -320,10 +327,11 @@ def iter_bytes(self, chunk_size: int = 1024): if __name__ == "__main__": # Example usage demonstrating OpenAI-compatible interface tts_provider = OpenAIFMTTS() - + try: # Basic usage - ic.configureOutput(prefix='DEBUG| '); ic("Testing basic speech generation...") + ic.configureOutput(prefix='DEBUG| ') + ic("Testing basic speech generation...") audio_file = tts_provider.create_speech( input="Today is a wonderful day to build something people love!", model="gpt-4o-mini-tts", @@ -331,18 +339,22 @@ def iter_bytes(self, chunk_size: int = 1024): instructions="Speak in a cheerful and positive tone." ) print(f"Audio file generated: {audio_file}") - + # Streaming usage - ic.configureOutput(prefix='DEBUG| '); ic("Testing streaming response...") + ic.configureOutput(prefix='DEBUG| ') + ic("Testing streaming response...") with tts_provider.with_streaming_response().create( input="This is a streaming test.", voice="alloy", response_format="wav" ) as response: response.stream_to_file("streaming_test.wav") - ic.configureOutput(prefix='INFO| '); ic("Streaming audio saved to streaming_test.wav") - + ic.configureOutput(prefix='INFO| ') + ic("Streaming audio saved to streaming_test.wav") + except exceptions.FailedToGenerateResponseError as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Error: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Error: {e}") except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Unexpected error: {e}") \ No newline at end of file + ic.configureOutput(prefix='ERROR| ') + ic(f"Unexpected error: {e}") diff --git a/webscout/Provider/TTS/parler.py b/webscout/Provider/TTS/parler.py index 00f8835c..89f88d28 100644 --- a/webscout/Provider/TTS/parler.py +++ b/webscout/Provider/TTS/parler.py @@ -2,13 +2,15 @@ ## ParlerTTS Provider ## ################################################################################## import json +import pathlib import random import string -import time -import pathlib import tempfile +from typing import Optional + import httpx -from typing import Optional, Union, Dict, List +from litprinter import ic + from webscout import exceptions from webscout.litagent import LitAgent @@ -17,32 +19,31 @@ from .base import BaseTTSProvider except ImportError: # Handle direct execution - import sys import os + import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) - from webscout.Provider.TTS import utils from webscout.Provider.TTS.base import BaseTTSProvider class ParlerTTS(BaseTTSProvider): """ Text-to-speech provider using the Parler-TTS API (Hugging Face Spaces). - + Features: - High-fidelity speech generation - Controllable via simple text prompts (description) - Manual polling logic for robustness """ required_auth = False - + BASE_URL = "https://parler-tts-parler-tts.hf.space" - + # Request headers headers: dict[str, str] = { "User-Agent": LitAgent().random(), "origin": BASE_URL, "referer": f"{BASE_URL}/", } - + SUPPORTED_MODELS = ["parler-mini-v1", "parler-large-v1"] def __init__(self, timeout: int = 120, proxy: Optional[str] = None): @@ -58,8 +59,8 @@ def _generate_session_hash(self) -> str: return "".join(random.choices(string.ascii_lowercase + string.digits, k=10)) def tts( - self, - text: str, + self, + text: str, description: str = "A female speaker delivers a slightly expressive and animated speech with a moderate speed. The recording features a low-pitch voice and very clear audio.", use_large: bool = False, response_format: str = "wav", @@ -70,16 +71,18 @@ def tts( """ if not text: raise ValueError("Input text must be a non-empty string") - + session_hash = self._generate_session_hash() filename = pathlib.Path(tempfile.mktemp(suffix=f".{response_format}", dir=self.temp_dir)) - + if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"ParlerTTS: Generating speech for '{text[:20]}...'") + ic.configureOutput(prefix='DEBUG| ') + ic(f"ParlerTTS: Generating speech for '{text[:20]}...'") client_kwargs = {"headers": self.headers, "timeout": self.timeout} - if self.proxy: client_kwargs["proxy"] = self.proxy - + if self.proxy: + client_kwargs["proxy"] = self.proxy + try: with httpx.Client(**client_kwargs) as client: # Step 1: Join the queue @@ -92,23 +95,25 @@ def tts( "trigger_id": 8, "session_hash": session_hash } - + response = client.post(join_url, json=payload) response.raise_for_status() - + # Step 2: Poll for data data_url = f"{self.BASE_URL}/queue/data?session_hash={session_hash}" audio_url = None - + # Gradio Spaces can take time to wake up or process with client.stream("GET", data_url) as stream: for line in stream.iter_lines(): - if not line: continue + if not line: + continue if line.startswith("data: "): try: data = json.loads(line[6:]) - except json.JSONDecodeError: continue - + except json.JSONDecodeError: + continue + msg = data.get("msg") if msg == "process_completed": if data.get("success"): @@ -132,17 +137,20 @@ def tts( # Step 3: Download the audio file audio_response = client.get(audio_url) audio_response.raise_for_status() - + with open(filename, "wb") as f: f.write(audio_response.content) - + if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Speech generated successfully: {filename}") - + ic.configureOutput(prefix='DEBUG| ') + ic(f"Speech generated successfully: {filename}") + return filename.as_posix() except Exception as e: - if verbose: ic.configureOutput(prefix='DEBUG| '); ic(f"Error in ParlerTTS: {e}") + if verbose: + ic.configureOutput(prefix='DEBUG| ') + ic(f"Error in ParlerTTS: {e}") raise exceptions.FailedToGenerateResponseError(f"Failed to generate audio: {e}") def create_speech(self, input: str, **kwargs) -> str: @@ -152,6 +160,8 @@ def create_speech(self, input: str, **kwargs) -> str: tts = ParlerTTS() try: path = tts.tts("Testing Parler-TTS with manual polling.", verbose=True) - ic.configureOutput(prefix='INFO| '); ic(f"Saved to {path}") + ic.configureOutput(prefix='INFO| ') + ic(f"Saved to {path}") except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Error: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Error: {e}") diff --git a/webscout/Provider/TTS/qwen.py b/webscout/Provider/TTS/qwen.py index fe1be83f..1848bd9a 100644 --- a/webscout/Provider/TTS/qwen.py +++ b/webscout/Provider/TTS/qwen.py @@ -2,13 +2,15 @@ ## Qwen3-TTS Provider ## ################################################################################## import json +import pathlib import random import string -import time -import pathlib import tempfile +from typing import Optional + import httpx -from typing import Optional, Union, Dict, List +from litprinter import ic + from webscout import exceptions from webscout.litagent import LitAgent @@ -17,16 +19,15 @@ from .base import BaseTTSProvider except ImportError: # Handle direct execution - import sys import os + import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) - from webscout.Provider.TTS import utils from webscout.Provider.TTS.base import BaseTTSProvider class QwenTTS(BaseTTSProvider): """ Text-to-speech provider using the Qwen3-TTS API (Hugging Face Spaces). - + This provider follows the OpenAI TTS API structure with support for: - Multiple TTS models (mapped to Gradio fn_index) - 40+ high-quality voices across multiple languages @@ -35,19 +36,19 @@ class QwenTTS(BaseTTSProvider): - Streaming response simulation """ required_auth = False - + BASE_URL = "https://qwen-qwen3-tts-demo.hf.space" - + # Request headers headers: dict[str, str] = { "User-Agent": LitAgent().random(), "origin": BASE_URL, "referer": f"{BASE_URL}/", } - + # Override supported models SUPPORTED_MODELS = ["qwen3-tts"] - + # Supported voices SUPPORTED_VOICES = [ "cherry", "serena", "ethan", "chelsie", "momo", "vivian", "moon", "maia", @@ -57,7 +58,7 @@ class QwenTTS(BaseTTSProvider): "neil", "elias", "arthur", "nini", "ebona", "seren", "pip", "stella", "li", "marcus", "roy", "peter", "eric", "rocky", "kiki", "sunny", "jada", "dylan" ] - + # Voice mapping for API compatibility voice_mapping = { "cherry": "Cherry / 芊悦", @@ -114,7 +115,7 @@ class QwenTTS(BaseTTSProvider): def __init__(self, timeout: int = 60, proxy: Optional[str] = None): """ Initialize the QwenTTS client. - + Args: timeout (int): Request timeout in seconds proxy (str): Proxy configuration string @@ -130,10 +131,10 @@ def _generate_session_hash(self) -> str: return "".join(random.choices(string.ascii_lowercase + string.digits, k=10)) def tts( - self, - text: str, + self, + text: str, model: str = "qwen3-tts", - voice: str = "cherry", + voice: str = "cherry", response_format: str = "wav", language: str = "Auto / 自动", verbose: bool = True @@ -158,26 +159,27 @@ def tts( """ if not text or not isinstance(text, str): raise ValueError("Input text must be a non-empty string") - + voice = self.validate_voice(voice) qwen_voice = self.voice_mapping.get(voice, self.voice_mapping["cherry"]) - + # Create temporary file file_extension = f".{response_format}" filename = pathlib.Path(tempfile.mktemp(suffix=file_extension, dir=self.temp_dir)) - + session_hash = self._generate_session_hash() - + if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Joining queue for voice: {voice} ({qwen_voice})") - + ic.configureOutput(prefix='DEBUG| ') + ic(f"Joining queue for voice: {voice} ({qwen_voice})") + client_kwargs = { "headers": self.headers, "timeout": self.timeout } if self.proxy: client_kwargs["proxy"] = self.proxy - + try: with httpx.Client(**client_kwargs) as client: # Step 1: Join the queue @@ -189,23 +191,24 @@ def tts( "trigger_id": 7, "session_hash": session_hash } - + response = client.post(join_url, json=payload) response.raise_for_status() - + # Step 2: Poll for data (SSE) data_url = f"{self.BASE_URL}/gradio_api/queue/data?session_hash={session_hash}" audio_url = None - + with client.stream("GET", data_url) as stream: for line in stream.iter_lines(): - if not line: continue + if not line: + continue if line.startswith("data: "): try: data = json.loads(line[6:]) except json.JSONDecodeError: continue - + msg = data.get("msg") if msg == "process_completed": if data.get("success"): @@ -226,24 +229,26 @@ def tts( # Step 3: Download the audio file audio_response = client.get(audio_url) audio_response.raise_for_status() - + with open(filename, "wb") as f: f.write(audio_response.content) - + if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Speech generated successfully: {filename}") - + ic.configureOutput(prefix='DEBUG| ') + ic(f"Speech generated successfully: {filename}") + return filename.as_posix() except Exception as e: if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Error in QwenTTS: {e}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Error in QwenTTS: {e}") raise exceptions.FailedToGenerateResponseError(f"Failed to generate audio: {e}") def create_speech( self, input: str, - model: str = "qwen3-tts", + model: str = "qwen3-tts", voice: str = "cherry", response_format: str = "wav", verbose: bool = False, @@ -267,11 +272,11 @@ class StreamingResponseContextManager: """Context manager for streaming TTS responses.""" def __init__(self, tts_provider: QwenTTS): self.tts_provider = tts_provider - + def create(self, **kwargs): audio_file = self.tts_provider.create_speech(**kwargs) return StreamingResponse(audio_file) - + def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass @@ -279,14 +284,14 @@ class StreamingResponse: """Streaming response object for TTS audio.""" def __init__(self, audio_file: str): self.audio_file = audio_file - + def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass - + def stream_to_file(self, file_path: str): import shutil shutil.copy2(self.audio_file, file_path) - + def iter_bytes(self, chunk_size: int = 1024): with open(self.audio_file, 'rb') as f: while chunk := f.read(chunk_size): @@ -295,8 +300,11 @@ def iter_bytes(self, chunk_size: int = 1024): if __name__ == "__main__": qwen = QwenTTS() try: - ic.configureOutput(prefix='DEBUG| '); ic("Testing Qwen3-TTS...") + ic.configureOutput(prefix='DEBUG| ') + ic("Testing Qwen3-TTS...") path = qwen.create_speech(input="Hello, this is a test.", voice="jennifer", verbose=True) - ic.configureOutput(prefix='INFO| '); ic(f"Saved to {path}") + ic.configureOutput(prefix='INFO| ') + ic(f"Saved to {path}") except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Error: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Error: {e}") diff --git a/webscout/Provider/TTS/sherpa.py b/webscout/Provider/TTS/sherpa.py index fd36770f..4bb7be4b 100644 --- a/webscout/Provider/TTS/sherpa.py +++ b/webscout/Provider/TTS/sherpa.py @@ -2,13 +2,15 @@ ## SherpaTTS Provider ## ################################################################################## import json +import pathlib import random import string -import time -import pathlib import tempfile +from typing import Optional + import httpx -from typing import Optional, Union, Dict, List +from litprinter import ic + from webscout import exceptions from webscout.litagent import LitAgent @@ -17,16 +19,15 @@ from .base import BaseTTSProvider except ImportError: # Handle direct execution - import sys import os + import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) - from webscout.Provider.TTS import utils from webscout.Provider.TTS.base import BaseTTSProvider class SherpaTTS(BaseTTSProvider): """ Text-to-speech provider using the Next-gen Kaldi (Sherpa-ONNX) API. - + This provider follows the OpenAI TTS API structure with support for: - 50+ languages including English, Chinese, Cantonese, Arabic, French, etc. - Multiple ONNX-based models (Kokoro, Piper, Coqui, etc.) @@ -34,16 +35,16 @@ class SherpaTTS(BaseTTSProvider): - Multiple output formats """ required_auth = False - + BASE_URL = "https://k2-fsa-text-to-speech.hf.space" - + # Request headers headers: dict[str, str] = { "User-Agent": LitAgent().random(), "origin": BASE_URL, "referer": f"{BASE_URL}/", } - + SUPPORTED_MODELS = [ "csukuangfj/kokoro-en-v0_19|11 speakers", "csukuangfj/kitten-kitten-en-v0_1-fp16|8 speakers", @@ -96,7 +97,7 @@ class SherpaTTS(BaseTTSProvider): "csukuangfj/vits-vctk|109 speakers", "csukuangfj/vits-ljs|1 speaker" ] - + LANGUAGES = [ "English", "Chinese (Mandarin, 普通话)", "Chinese+English", "Persian+English", "Cantonese (粤语)", "Min-nan (闽南话)", "Arabic", "Afrikaans", "Bengali", @@ -123,8 +124,8 @@ def _generate_session_hash(self) -> str: return "".join(random.choices(string.ascii_lowercase + string.digits, k=11)) def tts( - self, - text: str, + self, + text: str, language: str = "English", model_choice: str = "csukuangfj/kokoro-en-v0_19|11 speakers", speaker_id: str = "0", @@ -146,18 +147,20 @@ def tts( """ if not text: raise ValueError("Input text must be a non-empty string") - + model_choice = self.validate_model(model_choice) - + session_hash = self._generate_session_hash() filename = pathlib.Path(tempfile.mktemp(suffix=f".{response_format}", dir=self.temp_dir)) - + if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"SherpaTTS: Generating speech for '{text[:20]}...' using {language}/{model_choice}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"SherpaTTS: Generating speech for '{text[:20]}...' using {language}/{model_choice}") client_kwargs = {"headers": self.headers, "timeout": self.timeout} - if self.proxy: client_kwargs["proxy"] = self.proxy - + if self.proxy: + client_kwargs["proxy"] = self.proxy + try: with httpx.Client(**client_kwargs) as client: # Step 1: Join the queue @@ -169,22 +172,24 @@ def tts( "trigger_id": 9, "session_hash": session_hash } - + response = client.post(join_url, json=payload) response.raise_for_status() - + # Step 2: Poll for data data_url = f"{self.BASE_URL}/gradio_api/queue/data?session_hash={session_hash}" audio_url = None - + with client.stream("GET", data_url) as stream: for line in stream.iter_lines(): - if not line: continue + if not line: + continue if line.startswith("data: "): try: data = json.loads(line[6:]) - except json.JSONDecodeError: continue - + except json.JSONDecodeError: + continue + msg = data.get("msg") if msg == "process_completed": if data.get("success"): @@ -205,17 +210,20 @@ def tts( # Step 3: Download the audio file audio_response = client.get(audio_url) audio_response.raise_for_status() - + with open(filename, "wb") as f: f.write(audio_response.content) - + if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Speech generated successfully: {filename}") - + ic.configureOutput(prefix='DEBUG| ') + ic(f"Speech generated successfully: {filename}") + return filename.as_posix() except Exception as e: - if verbose: ic.configureOutput(prefix='DEBUG| '); ic(f"Error in SherpaTTS: {e}") + if verbose: + ic.configureOutput(prefix='DEBUG| ') + ic(f"Error in SherpaTTS: {e}") raise exceptions.FailedToGenerateResponseError(f"Failed to generate audio: {e}") def create_speech(self, input: str, **kwargs) -> str: @@ -251,6 +259,8 @@ def iter_bytes(self, chunk_size: int = 1024): tts = SherpaTTS() try: path = tts.tts("This is a Sherpa-ONNX test.", verbose=True) - ic.configureOutput(prefix='INFO| '); ic(f"Result: {path}") + ic.configureOutput(prefix='INFO| ') + ic(f"Result: {path}") except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Error: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Error: {e}") diff --git a/webscout/Provider/TTS/speechma.py b/webscout/Provider/TTS/speechma.py index bed1120d..5623dc13 100644 --- a/webscout/Provider/TTS/speechma.py +++ b/webscout/Provider/TTS/speechma.py @@ -1,23 +1,25 @@ ################################################################################## ## Modified version of code written by t.me/infip1217 ## ################################################################################## -import time -import requests import pathlib import tempfile +import time +from concurrent.futures import ThreadPoolExecutor, as_completed from io import BytesIO + +import requests +from litprinter import ic + from webscout import exceptions from webscout.litagent import LitAgent -from litprinter import ic -from concurrent.futures import ThreadPoolExecutor, as_completed try: from . import utils from .base import BaseTTSProvider except ImportError: # Handle direct execution - import sys import os + import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from webscout.Provider.TTS import utils from webscout.Provider.TTS.base import BaseTTSProvider @@ -25,7 +27,7 @@ class SpeechMaTTS(BaseTTSProvider): """ Text-to-speech provider using the SpeechMa API with OpenAI-compatible interface. - + This provider follows the OpenAI TTS API structure with support for: - Multiple TTS models (gpt-4o-mini-tts, tts-1, tts-1-hd) - Multilingual voices with pitch and rate control @@ -34,7 +36,7 @@ class SpeechMaTTS(BaseTTSProvider): - Streaming support """ required_auth = False - + # Request headers headers = { "authority": "speechma.com", @@ -43,10 +45,10 @@ class SpeechMaTTS(BaseTTSProvider): "content-type": "application/json", **LitAgent().generate_fingerprint() } - + # SpeechMa doesn't support different models - set to None SUPPORTED_MODELS = None - + # All supported voices from SpeechMa API SUPPORTED_VOICES = [ "aditi", "amy", "astrid", "bianca", "carla", "carmen", "celine", "chant", @@ -77,7 +79,7 @@ class SpeechMaTTS(BaseTTSProvider): "michelle", "roger", "libby", "ryan", "sonia", "thomas", "natasha", "william", "clara", "liam" ] - + # Voice mapping for SpeechMa API compatibility (lowercase keys for all voices) voice_mapping = { # Standard voices @@ -236,7 +238,7 @@ class SpeechMaTTS(BaseTTSProvider): def __init__(self, timeout: int = 20, proxies: dict = None): """ Initialize the SpeechMa TTS client. - + Args: timeout (int): Request timeout in seconds proxies (dict): Proxy configuration @@ -264,7 +266,7 @@ def create_speech( ) -> bytes: """ Create speech from text using OpenAI-compatible interface. - + Args: input (str): The text to convert to speech voice (str): Voice to use for generation @@ -273,10 +275,10 @@ def create_speech( speed (float): Speed of speech (0.25 to 4.0) instructions (str): Voice instructions (not used by SpeechMa) **kwargs: Additional parameters (pitch, rate for SpeechMa compatibility) - + Returns: bytes: Audio data - + Raises: ValueError: If input parameters are invalid exceptions.FailedToGenerateResponseError: If generation fails @@ -286,21 +288,21 @@ def create_speech( raise ValueError("Input text must be a non-empty string") if len(input) > 10000: raise ValueError("Input text exceeds maximum allowed length of 10,000 characters") - + model = self.validate_model(model or self.default_model) voice = self.validate_voice(voice) response_format = self.validate_format(response_format) - + # Convert speed to SpeechMa rate parameter rate = int((speed - 1.0) * 10) # Convert 0.25-4.0 to -7.5 to 30, clamp to -10 to 10 rate = max(-10, min(10, rate)) - + # Extract SpeechMa-specific parameters pitch = kwargs.get('pitch', 0) - + # Map voice to SpeechMa format speechma_voice = self.voice_mapping.get(voice, self.all_voices.get(voice.title(), "voice-116")) - + # Prepare payload payload = { "text": input, @@ -309,7 +311,7 @@ def create_speech( "rate": rate, "volume": 100 } - + try: response = self.session.post( self.api_url, @@ -318,40 +320,40 @@ def create_speech( timeout=self.timeout ) response.raise_for_status() - + # Validate audio response content_type = response.headers.get('content-type', '').lower() - if ('audio' in content_type or - response.content.startswith(b'\xff\xfb') or - response.content.startswith(b'ID3') or + if ('audio' in content_type or + response.content.startswith(b'\xff\xfb') or + response.content.startswith(b'ID3') or b'LAME' in response.content[:100]): return response.content else: raise exceptions.FailedToGenerateResponseError( f"Unexpected response format. Content-Type: {content_type}" ) - + except requests.exceptions.RequestException as e: raise exceptions.FailedToGenerateResponseError(f"API request failed: {e}") def with_streaming_response(self): """ Return a context manager for streaming responses. - + Returns: SpeechMaStreamingResponse: Context manager for streaming """ return SpeechMaStreamingResponse(self) def tts( - self, - text: str, + self, + text: str, model: str = None, - voice: str = "emma", + voice: str = "emma", response_format: str = "mp3", instructions: str = None, - pitch: int = 0, - rate: int = 0, + pitch: int = 0, + rate: int = 0, verbose: bool = True ) -> str: """ @@ -379,18 +381,18 @@ def tts( raise ValueError("Input text must be a non-empty string") if len(text) > 10000: raise ValueError("Input text exceeds maximum allowed length of 10,000 characters") - + # Validate model, voice, and format using base class methods model = self.validate_model(model or self.default_model) voice = self.validate_voice(voice) response_format = self.validate_format(response_format) - + # Map voice to SpeechMa API format speechma_voice = self.voice_mapping.get(voice, voice) if speechma_voice not in self.all_voices.values(): # Fallback to legacy voice mapping speechma_voice = self.all_voices.get(voice.title(), self.all_voices.get("Emma", "voice-116")) - + # Create temporary file with appropriate extension file_extension = f".{response_format}" if response_format != "pcm" else ".wav" filename = pathlib.Path(tempfile.mktemp(suffix=file_extension, dir=self.temp_dir)) @@ -398,10 +400,14 @@ def tts( # Split text into sentences using the utils module for better processing sentences = utils.split_sentences(text) if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Processing {len(sentences)} sentences") - ic.configureOutput(prefix='DEBUG| '); ic(f"Model: {model}") - ic.configureOutput(prefix='DEBUG| '); ic(f"Voice: {voice} -> {speechma_voice}") - ic.configureOutput(prefix='DEBUG| '); ic(f"Format: {response_format}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Processing {len(sentences)} sentences") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Model: {model}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Voice: {voice} -> {speechma_voice}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Format: {response_format}") def generate_audio_for_chunk(part_text: str, part_number: int): """ @@ -441,12 +447,13 @@ def generate_audio_for_chunk(part_text: str, part_number: int): # Check if response is audio data content_type = response.headers.get('content-type', '').lower() - if ('audio' in content_type or - response.content.startswith(b'\xff\xfb') or - response.content.startswith(b'ID3') or + if ('audio' in content_type or + response.content.startswith(b'\xff\xfb') or + response.content.startswith(b'ID3') or b'LAME' in response.content[:100]): if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Chunk {part_number} processed successfully") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Chunk {part_number} processed successfully") return part_number, response.content else: raise exceptions.FailedToGenerateResponseError( @@ -460,7 +467,8 @@ def generate_audio_for_chunk(part_text: str, part_number: int): f"Failed to generate audio for chunk {part_number} after {max_retries} retries: {e}" ) if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Retrying chunk {part_number} (attempt {retry_count + 1})") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Retrying chunk {part_number} (attempt {retry_count + 1})") time.sleep(1) # Brief delay before retry # Process chunks concurrently for better performance @@ -471,14 +479,15 @@ def generate_audio_for_chunk(part_text: str, part_number: int): executor.submit(generate_audio_for_chunk, sentence, i): i for i, sentence in enumerate(sentences) } - + for future in as_completed(future_to_chunk): try: chunk_number, audio_data = future.result() audio_chunks.append((chunk_number, audio_data)) except Exception as e: if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Error processing chunk: {e}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Error processing chunk: {e}") raise else: # Single sentence, process directly @@ -494,7 +503,8 @@ def generate_audio_for_chunk(part_text: str, part_number: int): with open(filename, 'wb') as f: f.write(combined_audio) if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Audio saved to: {filename}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Audio saved to: {filename}") return filename.as_posix() except IOError as e: raise exceptions.FailedToGenerateResponseError(f"Failed to save audio file: {e}") @@ -502,16 +512,16 @@ def generate_audio_for_chunk(part_text: str, part_number: int): class SpeechMaStreamingResponse: """Context manager for streaming SpeechMa TTS responses.""" - + def __init__(self, client: SpeechMaTTS): self.client = client - + def __enter__(self): return self - + def __exit__(self, exc_type, exc_val, exc_tb): pass - + def create_speech( self, input: str, @@ -524,10 +534,10 @@ def create_speech( ): """ Create speech with streaming response simulation. - + Note: SpeechMa doesn't support true streaming, so this returns the complete audio data wrapped in a BytesIO object. - + Args: input (str): Text to convert to speech voice (str): Voice to use @@ -536,7 +546,7 @@ def create_speech( speed (float): Speech speed instructions (str): Voice instructions **kwargs: Additional parameters - + Returns: BytesIO: Audio data stream """ @@ -556,7 +566,7 @@ def create_speech( if __name__ == "__main__": # Initialize the SpeechMa TTS client speechma = SpeechMaTTS() - + # Example 1: Basic usage with legacy method print("=== Example 1: Basic TTS ===") text = "Hello, this is a test of the SpeechMa text-to-speech API." @@ -565,7 +575,7 @@ def create_speech( print(f"Audio saved to: {audio_file}") except Exception as e: print(f"Error: {e}") - + # Example 2: OpenAI-compatible interface print("\n=== Example 2: OpenAI-compatible interface ===") try: @@ -577,14 +587,14 @@ def create_speech( speed=1.2 ) print(f"Generated {len(audio_data)} bytes of audio data") - + # Save to file with open("openai_compatible_test.mp3", "wb") as f: f.write(audio_data) print("Audio saved to: openai_compatible_test.mp3") except Exception as e: print(f"Error: {e}") - + # Example 3: Streaming response context manager print("\n=== Example 3: Streaming response ===") try: @@ -598,14 +608,14 @@ def create_speech( print(f"Streamed {len(audio_data)} bytes of audio data") except Exception as e: print(f"Error: {e}") - + # Example 4: Voice and model validation print("\n=== Example 4: Parameter validation ===") try: # Test supported voices print("Supported voices:", speechma.SUPPORTED_VOICES[:5], "...") print("Supported models:", speechma.SUPPORTED_MODELS) - + # Test with different parameters audio_file = speechma.tts( text="Testing different voice parameters.", @@ -617,4 +627,4 @@ def create_speech( ) print(f"Audio with custom parameters saved to: {audio_file}") except Exception as e: - print(f"Error: {e}") \ No newline at end of file + print(f"Error: {e}") diff --git a/webscout/Provider/TTS/streamElements.py b/webscout/Provider/TTS/streamElements.py index e4e55ca9..8341e914 100644 --- a/webscout/Provider/TTS/streamElements.py +++ b/webscout/Provider/TTS/streamElements.py @@ -1,16 +1,20 @@ -import time -import requests import pathlib -import urllib.parse import tempfile -from typing import Union +import time +import urllib.parse +from concurrent.futures import ThreadPoolExecutor, as_completed from io import BytesIO + +import requests +from litprinter import ic + from webscout import exceptions from webscout.litagent import LitAgent -from concurrent.futures import ThreadPoolExecutor, as_completed + from . import utils from .base import BaseTTSProvider + class StreamElements(BaseTTSProvider): """ @@ -21,7 +25,7 @@ class StreamElements(BaseTTSProvider): required_auth = False - + # Supported voices @@ -171,14 +175,17 @@ def generate_audio_for_chunk(part_text: str, part_number: int): # Check if the request was successful if response.ok and response.status_code == 200: if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Chunk {part_number} processed successfully") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Chunk {part_number} processed successfully") return part_number, response.content else: if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"No data received for chunk {part_number}. Retrying...") + ic.configureOutput(prefix='DEBUG| ') + ic(f"No data received for chunk {part_number}. Retrying...") except requests.RequestException as e: if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Error for chunk {part_number}: {e}. Retrying...") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Error for chunk {part_number}: {e}. Retrying...") time.sleep(1) try: # Using ThreadPoolExecutor to handle requests concurrently @@ -196,25 +203,29 @@ def generate_audio_for_chunk(part_text: str, part_number: int): audio_chunks[part_number] = audio_data # Store the audio data in correct sequence except Exception as e: if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Failed to generate audio for chunk {chunk_num}: {e}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Failed to generate audio for chunk {chunk_num}: {e}") # Combine audio chunks in the correct sequence combined_audio = BytesIO() for part_number in sorted(audio_chunks.keys()): combined_audio.write(audio_chunks[part_number]) if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Added chunk {part_number} to the combined file.") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Added chunk {part_number} to the combined file.") # Save the combined audio data to a single file with open(filename, 'wb') as f: f.write(combined_audio.getvalue()) if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Final Audio Saved as {filename}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Final Audio Saved as {filename}") return filename.as_posix() except requests.exceptions.RequestException as e: if verbose: - ic.configureOutput(prefix='DEBUG| '); ic(f"Failed to perform the operation: {e}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Failed to perform the operation: {e}") raise exceptions.FailedToGenerateResponseError( f"Failed to perform the operation: {e}" ) @@ -224,6 +235,8 @@ def generate_audio_for_chunk(part_text: str, part_number: int): streamelements = StreamElements() text = "This is a test of the StreamElements text-to-speech API. It supports multiple sentences and advanced logging." - ic.configureOutput(prefix='DEBUG| '); ic("Generating audio...") + ic.configureOutput(prefix='DEBUG| ') + ic("Generating audio...") audio_file = streamelements.tts(text, voice="Mathieu") - ic.configureOutput(prefix='INFO| '); ic(f"Audio saved to: {audio_file}") + ic.configureOutput(prefix='INFO| ') + ic(f"Audio saved to: {audio_file}") diff --git a/webscout/Provider/TTS/utils.py b/webscout/Provider/TTS/utils.py index db7acec0..47735c61 100644 --- a/webscout/Provider/TTS/utils.py +++ b/webscout/Provider/TTS/utils.py @@ -1,13 +1,13 @@ """ Text processing utilities for TTS providers. """ -from typing import Union, List, Dict, Tuple, Set, Optional, Pattern import re +from typing import Dict, List, Pattern, Set, Tuple class SentenceTokenizer: """Advanced sentence tokenizer with support for complex cases and proper formatting.""" - + def __init__(self) -> None: # Common abbreviations by category self.TITLES: Set[str] = { @@ -15,31 +15,31 @@ def __init__(self) -> None: 'hon', 'pres', 'gov', 'atty', 'supt', 'det', 'rev', 'col','maj', 'gen', 'capt', 'cmdr', 'lt', 'sgt', 'cpl', 'pvt' } - + self.ACADEMIC: Set[str] = { 'ph.d', 'phd', 'm.d', 'md', 'b.a', 'ba', 'm.a', 'ma', 'd.d.s', 'dds', 'm.b.a', 'mba', 'b.sc', 'bsc', 'm.sc', 'msc', 'llb', 'll.b', 'bl' } - + self.ORGANIZATIONS: Set[str] = { 'inc', 'ltd', 'co', 'corp', 'llc', 'llp', 'assn', 'bros', 'plc', 'cos', 'intl', 'dept', 'est', 'dist', 'mfg', 'div' } - + self.MONTHS: Set[str] = { 'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec' } - + self.UNITS: Set[str] = { 'oz', 'pt', 'qt', 'gal', 'ml', 'cc', 'km', 'cm', 'mm', 'ft', 'in', 'kg', 'lb', 'lbs', 'hz', 'khz', 'mhz', 'ghz', 'kb', 'mb', 'gb', 'tb' } - + self.TECHNOLOGY: Set[str] = { 'v', 'ver', 'app', 'sys', 'dir', 'exe', 'lib', 'api', 'sdk', 'url', 'cpu', 'gpu', 'ram', 'rom', 'hdd', 'ssd', 'lan', 'wan', 'sql', 'html' } - + self.MISC: Set[str] = { 'vs', 'etc', 'ie', 'eg', 'no', 'al', 'ca', 'cf', 'pp', 'est', 'st', 'approx', 'appt', 'apt', 'dept', 'depts', 'min', 'max', 'avg' @@ -60,13 +60,13 @@ def __init__(self) -> None: self.NUMBER_PATTERN: str = ( r'\d+(?:\.\d+)?(?:%|°|km|cm|mm|m|kg|g|lb|ft|in|mph|kmh|hz|mhz|ghz)?' ) - + # Quote and bracket pairs self.QUOTE_PAIRS: Dict[str, str] = { - '"': '"', "'": "'", '"': '"', "「": "」", "『": "』", - "«": "»", "‹": "›", "'": "'", "‚": "'" + '"': '"', "'": "'", "「": "」", "『": "』", + "«": "»", "‹": "›", "‚": "'" } - + self.BRACKETS: Dict[str, str] = { '(': ')', '[': ']', '{': '}', '⟨': '⟩', '「': '」', '『': '』', '【': '】', '〖': '〗', '「': '」' @@ -84,14 +84,14 @@ def _compile_patterns(self) -> None: (?: # Standard endings with optional quotes/brackets (?<=[.!?])[\"\'\)\]\}»›」』\s]* - + # Ellipsis |(?:\.{2,}|…) - + # Asian-style endings |(?<=[。!?」』】\s]) ) - + # Must be followed by whitespace and capital letter or number (?=\s+(?:[A-Z0-9]|["'({[\[「『《‹〈][A-Z])) ''', @@ -169,30 +169,30 @@ def _restore_formatting(self, sentences: List[str]) -> List[str]: for sentence in sentences: # Restore dots in abbreviations sentence = sentence.replace('__DOT__', '.') - + # Restore paragraph breaks sentence = sentence.replace('__PARA__', '\n\n') - + # Clean up whitespace sentence = re.sub(r'\s+', ' ', sentence).strip() - + # Capitalize first letter if it's lowercase and not an abbreviation words = sentence.split() if words and words[0].lower() not in self.all_abbreviations: sentence = sentence[0].upper() + sentence[1:] - + if sentence: restored.append(sentence) - + return restored def tokenize(self, text: str) -> List[str]: """ Split text into sentences while handling complex cases. - + Args: text (str): Input text to split into sentences. - + Returns: List[str]: List of properly formatted sentences. """ @@ -201,31 +201,31 @@ def tokenize(self, text: str) -> List[str]: # Step 1: Protect special cases protected_text, placeholders = self._protect_special_cases(text) - + # Step 2: Normalize whitespace protected_text = self._normalize_whitespace(protected_text) - + # Step 3: Handle abbreviations protected_text = self._handle_abbreviations(protected_text) - + # Step 4: Split into potential sentences potential_sentences = self.SENTENCE_END.split(protected_text) - + # Step 5: Process and restore formatting sentences = self._restore_formatting(potential_sentences) - + # Step 6: Restore special cases sentences = [self._restore_special_cases(s, placeholders) for s in sentences] - + # Step 7: Post-process sentences final_sentences = [] current_sentence = [] - + for sentence in sentences: # Skip empty sentences if not sentence.strip(): continue - + # Check if sentence might be continuation of previous if current_sentence and sentence[0].islower(): current_sentence.append(sentence) @@ -233,21 +233,21 @@ def tokenize(self, text: str) -> List[str]: if current_sentence: final_sentences.append(' '.join(current_sentence)) current_sentence = [sentence] - + # Add last sentence if exists if current_sentence: final_sentences.append(' '.join(current_sentence)) - + return final_sentences def split_sentences(text: str) -> List[str]: """ Convenience function to split text into sentences using SentenceTokenizer. - + Args: text (str): Input text to split into sentences. - + Returns: List[str]: List of properly formatted sentences. """ @@ -258,19 +258,19 @@ def split_sentences(text: str) -> List[str]: if __name__ == "__main__": # Test text with various challenging cases test_text: str = """ - Dr. Smith (Ph.D., M.D.) visited Washington D.C. on Jan. 20, 2024! He met with Prof. Johnson at 3:30 p.m. - They discussed A.I. and machine learning... "What about the U.S. market?" asked Dr. Smith. + Dr. Smith (Ph.D., M.D.) visited Washington D.C. on Jan. 20, 2024! He met with Prof. Johnson at 3:30 p.m. + They discussed A.I. and machine learning... "What about the U.S. market?" asked Dr. Smith. The meeting ended at 5 p.m. Later, they went to Mr. Wilson's house (located at 123 Main St.) for dinner. - + Visit our website at https://www.example.com or email us at test@example.com! The temperature was 72.5°F (22.5°C). The company's Q3 2023 revenue was $12.5M USD. - + 「これは日本語の文章です。」This is a mixed-language text! How cool is that? - + Some technical specs: CPU: 3.5GHz, RAM: 16GB, Storage: 2TB SSD. Common abbreviations: etc., i.e., e.g., vs., cf., approx. 100 units. """ - + # Process and print each sentence sentences: List[str] = split_sentences(test_text) print("Detected sentences:") diff --git a/webscout/Provider/TextPollinationsAI.py b/webscout/Provider/TextPollinationsAI.py index 9c45cdc0..8661512e 100644 --- a/webscout/Provider/TextPollinationsAI.py +++ b/webscout/Provider/TextPollinationsAI.py @@ -1,13 +1,15 @@ -import requests import json -from typing import Union, Any, Dict, Generator, Optional, List +from typing import Any, Dict, Generator, List, Optional, Union + +import requests -from webscout.AIutel import Optimizers, Conversation, AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent as Lit + class TextPollinationsAI(Provider): """ A class to interact with the Pollinations AI API. @@ -15,7 +17,7 @@ class TextPollinationsAI(Provider): required_auth = False _models_url = "https://text.pollinations.ai/models" - + # Static list as fallback AVAILABLE_MODELS = [ "deepseek", @@ -39,14 +41,14 @@ class TextPollinationsAI(Provider): def __init__(self, is_conversation: bool = True, - max_tokens: int = 8096, + max_tokens: int = 8096, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "openai", system_prompt: str = "You are a helpful AI assistant.", ): @@ -63,7 +65,7 @@ def __init__(self, # Fetch latest models dynamically to ensure we have the most up-to-date list self.update_available_models() - + if model not in self.AVAILABLE_MODELS: # warn or just allow it? allowing it for flexibility if model not in self.AVAILABLE_MODELS: @@ -119,11 +121,12 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, tools: Optional[List[Dict[str, Any]]] = None, tool_choice: Optional[Dict[str, Any]] = None, - ) -> Union[Dict[str, Any], Generator[Any, None, None]]: + **kwargs: Any, + ) -> Response: """Chat with AI""" conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: @@ -157,7 +160,7 @@ def for_stream(): timeout=self.timeout ) response.raise_for_status() - + streaming_text = "" processed_stream = sanitize_stream( data=response.iter_content(chunk_size=None), @@ -190,7 +193,7 @@ def for_stream(): self.last_response.update(dict(text=streaming_text)) if streaming_text: self.conversation.update_chat_history(prompt, streaming_text) - + except Exception as e: raise exceptions.FailedToGenerateResponseError(f"Stream request failed: {e}") from e @@ -204,7 +207,7 @@ def for_non_stream(): timeout=self.timeout ) response.raise_for_status() - + # Use sanitize_stream to parse the non-streaming JSON response processed_stream = sanitize_stream( data=response.text, @@ -215,19 +218,23 @@ def for_non_stream(): ) # Extract the single result resp_json = next(processed_stream, None) - + # Check for standard OpenAI response structure if resp_json and 'choices' in resp_json and len(resp_json['choices']) > 0: - + choice = resp_json['choices'][0] + content = choice.get('message', {}).get('content') + tool_calls = choice.get('message', {}).get('tool_calls') + result = content if content else (tool_calls if tool_calls else "") + self.last_response = result self.conversation.update_chat_history(prompt, content or "") - + if raw: return content if content else (json.dumps(tool_calls) if tool_calls else "") return result - + else: - return {} + return {} except Exception as e: raise exceptions.FailedToGenerateResponseError(f"Non-stream request failed: {e}") from e @@ -238,7 +245,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, tools: Optional[List[Dict[str, Any]]] = None, tool_choice: Optional[Dict[str, Any]] = None, @@ -281,25 +288,25 @@ def get_message(self, response: dict) -> str: print("-" * 80) print(f"{'Model':<50} {'Status':<10} {'Response'}") print("-" * 80) - + # Test only a subset to be fast - test_models = ["openai", "gemini"] - + test_models = ["openai", "gemini"] + for model in test_models: try: print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True) test_ai = TextPollinationsAI(model=model, timeout=60) - + # Non-stream test start_response = test_ai.chat("Hello!", stream=False) - if start_response: + if start_response and isinstance(start_response, str): status = "✓" display = start_response[:30] + "..." else: status = "✗" - display = "Empty" - + display = "Empty or invalid type" + print(f"\r{model:<50} {status:<10} {display}") - + except Exception as e: - print(f"\r{model:<50} {'✗':<10} {str(e)[:50]}") \ No newline at end of file + print(f"\r{model:<50} {'✗':<10} {str(e)[:50]}") diff --git a/webscout/Provider/TogetherAI.py b/webscout/Provider/TogetherAI.py index 9d6db3ba..3dc21783 100644 --- a/webscout/Provider/TogetherAI.py +++ b/webscout/Provider/TogetherAI.py @@ -31,25 +31,25 @@ def get_models(cls, api_key: str = None): "Authorization": f"Bearer {api_key}", "Content-Type": "application/json" } - + response = session.get( "https://api.together.xyz/v1/models", headers=headers, impersonate="chrome110" ) - + if response.status_code != 200: return cls.AVAILABLE_MODELS - + data = response.json() # Together API returns a list of model objects if isinstance(data, list): # Filter for chat/language models if possible, or just return all IDs # The API returns objects with 'id', 'type', etc. return [model["id"] for model in data if isinstance(model, dict) and "id" in model] - + return cls.AVAILABLE_MODELS - + except Exception: return cls.AVAILABLE_MODELS diff --git a/webscout/Provider/TwoAI.py b/webscout/Provider/TwoAI.py index faed86bd..9e8b681c 100644 --- a/webscout/Provider/TwoAI.py +++ b/webscout/Provider/TwoAI.py @@ -1,16 +1,13 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError -import json import base64 -from typing import Any, Dict, Optional, Generator, Union -import re # Import re for parsing SSE +import json +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions -from webscout.litagent import LitAgent +from webscout.AIbase import Provider +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream class TwoAI(Provider): @@ -297,7 +294,7 @@ def chat( aggregated_text += response_dict["text"] elif isinstance(response_dict, str): aggregated_text += response_dict - + return aggregated_text def get_message(self, response: dict) -> str: @@ -310,4 +307,4 @@ def get_message(self, response: dict) -> str: ai = TwoAI(api_key="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VySWQiOiJzanl2OHJtZGxDZDFnQ2hQdGxzZHdxUlVteXkyIiwic291cmNlIjoiRmlyZWJhc2UiLCJpYXQiOjE3NTc4NTEyMzYsImV4cCI6MTc1Nzg1MjEzNn0.ilTYrHRdN3_cme6VW3knWWfbypY_n_gsUe9DeDhEwrM", model="sutra-v2", temperature=0.7) response = ai.chat("Write a poem about AI in the style of Shakespeare.") for chunk in response: - print(chunk, end="", flush=True) \ No newline at end of file + print(chunk, end="", flush=True) diff --git a/webscout/Provider/TypliAI.py b/webscout/Provider/TypliAI.py index b9a5eb18..b285c891 100644 --- a/webscout/Provider/TypliAI.py +++ b/webscout/Provider/TypliAI.py @@ -1,14 +1,15 @@ import random import string -from typing import Union, Any, Dict, Generator +from typing import Any, Dict, Generator, Union + from curl_cffi import CurlError from curl_cffi.requests import Session -# from curl_cffi.const import CurlHttpVersion # Not strictly needed if using default -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider + from webscout import exceptions +from webscout.AIbase import Provider + +# from curl_cffi.const import CurlHttpVersion # Not strictly needed if using default +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent @@ -33,10 +34,10 @@ class TypliAI(Provider): """ required_auth = False AVAILABLE_MODELS = [ - "openai/gpt-4.1-mini", + "openai/gpt-4.1-mini", "openai/gpt-4.1", - "openai/gpt-5-mini", - "openai/gpt-5.2", + "openai/gpt-5-mini", + "openai/gpt-5.2", "openai/gpt-5.2-pro", "google/gemini-2.5-flash", "anthropic/claude-haiku-4-5", @@ -309,4 +310,4 @@ def get_message(self, response: dict) -> str: for chunk in response: print(chunk, end="", flush=True) except Exception as e: - print(f"An error occurred: {e}") \ No newline at end of file + print(f"An error occurred: {e}") diff --git a/webscout/Provider/UNFINISHED/ChatHub.py b/webscout/Provider/UNFINISHED/ChatHub.py index a387e54f..6db4ddf7 100644 --- a/webscout/Provider/UNFINISHED/ChatHub.py +++ b/webscout/Provider/UNFINISHED/ChatHub.py @@ -1,13 +1,12 @@ -import requests import json -import os -from typing import Any, Dict, Optional, Generator, List, Union +from typing import Any, Dict, Generator, Union + +import requests -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream + class ChatHub(Provider): """ @@ -31,7 +30,7 @@ class ChatHub(Provider): def __init__( self, is_conversation: bool = True, - max_tokens: int = 2049, + max_tokens: int = 2049, timeout: int = 30, intro: str = None, filepath: str = None, @@ -39,7 +38,7 @@ def __init__( proxies: dict = {}, history_offset: int = 10250, act: str = None, - model: str = "sonar-online", + model: str = "sonar-online", ): """Initializes the ChatHub API client.""" self.url = "https://app.chathub.gg" @@ -53,8 +52,8 @@ def __init__( 'X-App-Id': 'web' } self.session = requests.Session() - self.session.headers.update(self.headers) - self.session.proxies.update(proxies) + self.session.headers.update(self.headers) + self.session.proxies.update(proxies) self.timeout = timeout self.last_response = {} @@ -127,7 +126,7 @@ def ask( def for_stream(): try: with requests.post(self.api_endpoint, headers=self.headers, json=data, stream=True, timeout=self.timeout) as response: - response.raise_for_status() + response.raise_for_status() streaming_text = "" # Use sanitize_stream for processing @@ -214,4 +213,4 @@ def get_message(self, response: dict) -> str: for chunk in response: print(chunk, end="", flush=True) except Exception as e: - print(f"An error occurred: {e}") \ No newline at end of file + print(f"An error occurred: {e}") diff --git a/webscout/Provider/UNFINISHED/ChutesAI.py b/webscout/Provider/UNFINISHED/ChutesAI.py index aaedffec..8c940e79 100644 --- a/webscout/Provider/UNFINISHED/ChutesAI.py +++ b/webscout/Provider/UNFINISHED/ChutesAI.py @@ -1,21 +1,29 @@ -import requests import json -import time -import uuid -from typing import List, Dict, Optional, Union, Generator, Any -import re import random +import re import string +import time +import uuid +from typing import Any, Dict, Generator, List, Optional, Union + +import cloudscraper +import requests from rich import print + from webscout.litagent.agent import LitAgent -import cloudscraper + # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, ) + # --- ChutesAI API Key Auto-Generator --- def generate_chutesai_api_key(): url = "https://chutes.ai/auth/start?/create" @@ -46,10 +54,10 @@ def generate_username(length=8): scraper = cloudscraper.create_scraper() response = scraper.post(url, headers=headers, data=data) print(f"[bold green]Status:[/] {response.status_code}") - + # Ensure response is decoded as UTF-8 response.encoding = 'utf-8' - + try: resp_json = response.json() except Exception: @@ -258,21 +266,21 @@ class ChutesAI(OpenAICompatibleProvider): def __init__(self, api_key: str = None,): self.timeout = None # Infinite timeout self.base_url = "https://llm.chutes.ai/v1/chat/completions" - + # Always generate a new API key, ignore any provided key print("[yellow]Generating new ChutesAI API key...[/]") self.api_key = generate_chutesai_api_key() - + if not self.api_key: print("[red]Failed to generate API key. Retrying...[/]") # Retry once more self.api_key = generate_chutesai_api_key() - + if not self.api_key: raise ValueError("Failed to generate ChutesAI API key after multiple attempts.") - + print(f"[green]Successfully generated API key: {self.api_key[:20]}...[/]") - + self.scraper = cloudscraper.create_scraper() self.headers = { "Authorization": f"Bearer {self.api_key}", @@ -292,12 +300,12 @@ def list(inner_self): try: # Example usage - always use generated API key client = ChutesAI() - + messages = [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "What is the capital of France?"} ] - + print("[cyan]Making API request...[/]") response = client.chat.completions.create( model="deepseek-ai/DeepSeek-V3-0324", @@ -311,7 +319,7 @@ def list(inner_self): else: chunk_dict = chunk.dict(exclude_none=True) print(f"[green]Response Chunk:[/] {chunk_dict}") - + except Exception as e: print(f"[red]Error: {e}[/]") - print("[yellow]If the issue persists, the ChutesAI service might be down or the API key generation method needs updating.[/]") \ No newline at end of file + print("[yellow]If the issue persists, the ChutesAI service might be down or the API key generation method needs updating.[/]") diff --git a/webscout/Provider/UNFINISHED/GizAI.py b/webscout/Provider/UNFINISHED/GizAI.py index 82098e23..106b2aac 100644 --- a/webscout/Provider/UNFINISHED/GizAI.py +++ b/webscout/Provider/UNFINISHED/GizAI.py @@ -1,28 +1,27 @@ -import os import base64 -import random import json -from typing import Union, Dict, Any, Optional, Generator +import os +import random +from typing import Any, Dict, Generator, Union from urllib import response from curl_cffi import CurlError -from curl_cffi.requests import Session from curl_cffi.const import CurlHttpVersion +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers from webscout.litagent import LitAgent + class GizAI(Provider): """ A class to interact with the GizAI API. - + Attributes: system_prompt (str): The system prompt to define the assistant's role. - + Examples: >>> from webscout.Provider.GizAI import GizAI >>> ai = GizAI() @@ -55,7 +54,7 @@ class GizAI(Provider): "phi-4", "qwq-32b" ] - + def __init__( self, is_conversation: bool = True, @@ -73,15 +72,15 @@ def __init__( """Initializes the GizAI API client.""" if model not in self.AVAILABLE_MODELS: raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}") - + self.api_url = "https://app.giz.ai/api/data/users/inferenceServer.infer" - + # Initialize LitAgent for user-agent generation self.agent = LitAgent() - + # Initialize curl_cffi Session self.session = Session() - + # Set up the headers self.headers = { "accept": "application/json, text/plain, */*", @@ -93,11 +92,11 @@ def __init__( "sec-fetch-mode": "cors", "sec-fetch-site": "same-origin" } - + # Update session headers and proxies self.session.headers.update(self.headers) self.session.proxies = proxies - + # Store configuration self.system_prompt = system_prompt self.is_conversation = is_conversation @@ -105,13 +104,13 @@ def __init__( self.timeout = timeout self.last_response = {} self.model = model - + self.__available_optimizers = ( method for method in dir(Optimizers) if callable(getattr(Optimizers, method)) and not method.startswith("__") ) - + Conversation.intro = ( AwesomePrompts().get_act( act, raise_not_found=True, default=None, case_insensitive=True @@ -119,22 +118,22 @@ def __init__( if act else intro or Conversation.intro ) - + self.conversation = Conversation( is_conversation, self.max_tokens_to_sample, filepath, update_file ) self.conversation.history_offset = history_offset - + def _generate_id(self, length: int = 21) -> str: """Generates a random URL-safe base64 string.""" random_bytes = os.urandom(length * 2) # Generate more bytes initially b64_encoded = base64.urlsafe_b64encode(random_bytes).decode('utf-8') return b64_encoded[:length] - + def _get_random_ip(self) -> str: """Generates a random IPv4 address string.""" return f"{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}" - + def ask( self, prompt: str, @@ -145,17 +144,17 @@ def ask( ) -> Dict[str, Any]: """ Sends a prompt to the GizAI API and returns the response. - + Args: prompt (str): The prompt to send to the API. stream (bool): Not supported by GizAI, kept for compatibility. raw (bool): Whether to return the raw response. optimizer (str): Optimizer to use for the prompt. conversationally (bool): Whether to generate the prompt conversationally. - + Returns: Dict[str, Any]: The API response. - + Examples: >>> ai = GizAI() >>> response = ai.ask("Tell me a joke!") @@ -168,12 +167,12 @@ def ask( ) else: raise Exception(f"Optimizer is not one of {self.__available_optimizers}") - + # Generate random IDs for request instance_id = self._generate_id() subscribe_id = self._generate_id() x_forwarded_for = self._get_random_ip() - + # Set up request body - GizAI doesn't support streaming request_body = { "model": "chat", @@ -189,10 +188,10 @@ def ask( "instanceId": instance_id, "subscribeId": subscribe_id } - + # Combine default headers with the dynamic x-forwarded-for header request_headers = {**self.headers, "x-forwarded-for": x_forwarded_for} - + try: # Use curl_cffi session post with impersonate response = self.session.post( @@ -204,7 +203,7 @@ def ask( http_version=CurlHttpVersion.V2_0 # Use HTTP/2 ) response.raise_for_status() # Check for HTTP errors - + # Process the response try: response_json = response.json() @@ -221,19 +220,19 @@ def ask( except json.JSONDecodeError: # Handle case where response is not valid JSON content = response.text - + # Update conversation history self.last_response = {"text": content} self.conversation.update_chat_history(prompt, content) - + return self.last_response if not raw else content - + except CurlError as e: raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {str(e)}") except Exception as e: error_text = getattr(e, 'response', None) and getattr(e.response, 'text', '') raise exceptions.FailedToGenerateResponseError(f"Request failed ({type(e).__name__}): {str(e)} - {error_text}") - + def chat( self, prompt: str, @@ -243,16 +242,16 @@ def chat( ) -> 'Generator[str, None, None]': """ Generates a response from the GizAI API. - + Args: prompt (str): The prompt to send to the API. stream (bool): Not supported by GizAI, kept for compatibility. optimizer (str): Optimizer to use for the prompt. conversationally (bool): Whether to generate the prompt conversationally. - + Returns: Generator[str, None, None]: The API response text as a generator. - + Examples: >>> ai = GizAI() >>> response = ai.chat("What's the weather today?") @@ -267,17 +266,17 @@ def chat( yield result else: return result - + def get_message(self, response: Union[dict, str]) -> str: """ Extracts the message from the API response. - + Args: response (Union[dict, str]): The API response. - + Returns: str: The message content. - + Examples: >>> ai = GizAI() >>> response = ai.ask("Tell me a joke!") @@ -292,4 +291,4 @@ def get_message(self, response: Union[dict, str]) -> str: ai = GizAI() response = ai.chat("Hello, how are you?", stream=True) for chunk in response: - print(chunk, end="", flush=True) \ No newline at end of file + print(chunk, end="", flush=True) diff --git a/webscout/Provider/UNFINISHED/Marcus.py b/webscout/Provider/UNFINISHED/Marcus.py index 7e8a6729..a2d390e2 100644 --- a/webscout/Provider/UNFINISHED/Marcus.py +++ b/webscout/Provider/UNFINISHED/Marcus.py @@ -1,13 +1,17 @@ -from curl_cffi.requests import Session +from typing import Any, Dict, Generator, Union + from curl_cffi import CurlError -import json -from typing import Union, Any, Dict, Optional, Generator +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) + class Marcus(Provider): """ @@ -35,14 +39,14 @@ def __init__( self.api_endpoint = "https://www.askmarcus.app/api/response" self.timeout = timeout self.last_response = {} - + self.headers = { 'content-type': 'application/json', 'accept': '*/*', 'origin': 'https://www.askmarcus.app', 'referer': 'https://www.askmarcus.app/chat', } - + # Update curl_cffi session headers and proxies self.session.headers.update(self.headers) self.session.proxies = proxies # Assign proxies directly @@ -102,7 +106,7 @@ def for_stream(): impersonate="chrome110" # Use a common impersonation profile ) response.raise_for_status() # Check for HTTP errors - + # Use sanitize_stream to decode bytes and yield text chunks processed_stream = sanitize_stream( data=response.iter_content(chunk_size=None), # Pass byte iterator @@ -139,7 +143,7 @@ def for_non_stream(): impersonate="chrome110" # Use a common impersonation profile ) response.raise_for_status() # Check for HTTP errors - + response_text_raw = response.text # Get raw text # Process the text using sanitize_stream (even though it's not streaming) diff --git a/webscout/Provider/UNFINISHED/Qodo.py b/webscout/Provider/UNFINISHED/Qodo.py index 456bbbbb..92eb7312 100644 --- a/webscout/Provider/UNFINISHED/Qodo.py +++ b/webscout/Provider/UNFINISHED/Qodo.py @@ -1,16 +1,15 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError -from typing import Any, Dict, Optional, Generator, Union import uuid -import json +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent + class QodoAI(Provider): """ A class to interact with the Qodo AI API. @@ -18,7 +17,7 @@ class QodoAI(Provider): AVAILABLE_MODELS = [ "gpt-4.1", - "gpt-4o", + "gpt-4o", "o3", "o4-mini", "claude-4-sonnet", @@ -56,21 +55,21 @@ def __init__( """Initializes the Qodo AI API client.""" if model not in self.AVAILABLE_MODELS: raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}") - + self.url = "https://api.cli.qodo.ai/v2/agentic/start-task" self.info_url = "https://api.cli.qodo.ai/v2/info/get-things" - + # Initialize LitAgent for user agent generation self.agent = LitAgent() self.fingerprint = self.agent.generate_fingerprint(browser) - + # Store API key self.api_key = api_key or "sk-dS7U-extxMWUxc8SbYYOuncqGUIE8-y2OY8oMCpu0eI-qnSUyH9CYWO_eAMpqwfMo7pXU3QNrclfZYMO0M6BJTM" - + # Generate session ID dynamically from API self.session_id = self._get_session_id() self.request_id = str(uuid.uuid4()) - + # Use the fingerprint for headers self.headers = { "Accept": "text/plain", @@ -83,7 +82,7 @@ def __init__( "Request-id": self.request_id, "User-Agent": self.fingerprint["user_agent"], } - + # Initialize curl_cffi Session self.session = Session() # Add Session-id to headers after getting it from API @@ -118,23 +117,23 @@ def __init__( def refresh_identity(self, browser: str = None): """ Refreshes the browser identity fingerprint. - + Args: browser: Specific browser to use for the new fingerprint """ browser = browser or self.fingerprint.get("browser_type", "chrome") self.fingerprint = self.agent.generate_fingerprint(browser) - + # Update headers with new fingerprint self.headers.update({ "Accept-Language": self.fingerprint["accept_language"], "User-Agent": self.fingerprint["user_agent"], }) - + # Update session headers for header, value in self.headers.items(): self.session.headers[header] = value - + return self.fingerprint def _build_payload(self, prompt: str): @@ -379,13 +378,13 @@ def _get_session_id(self) -> str: "User-Agent": self.fingerprint["user_agent"] if hasattr(self, 'fingerprint') else "axios/1.10.0", } temp_session.headers.update(temp_headers) - + response = temp_session.get( self.info_url, timeout=self.timeout if hasattr(self, 'timeout') else 30, impersonate="chrome110" ) - + if response.status_code == 200: data = response.json() session_id = data.get("session-id") @@ -406,12 +405,12 @@ def _get_session_id(self) -> str: "Usage: QodoAI(api_key='your_api_key_here')\n" "To get an API key, install Qodo CLI via: https://docs.qodo.ai/qodo-documentation/qodo-gen-cli/getting-started/setup-and-quickstart" ) - + # Fallback to generated session ID if API call fails from datetime import datetime today = datetime.now().strftime("%Y%m%d") return f"{today}-{str(uuid.uuid4())}" - + except exceptions.FailedToGenerateResponseError: # Re-raise our custom exceptions raise @@ -427,23 +426,22 @@ def _get_session_id(self) -> str: def refresh_session(self): """ Refreshes the session ID by calling the Qodo API. - + Returns: str: The new session ID """ - old_session_id = self.session_id self.session_id = self._get_session_id() - + # Update headers with new session ID self.headers["Session-id"] = self.session_id self.session.headers["Session-id"] = self.session_id - + return self.session_id def get_available_models(self) -> Dict[str, Any]: """ Get available models and info from Qodo API. - + Returns: Dict containing models, default_model, version, and session info """ @@ -453,7 +451,7 @@ def get_available_models(self) -> Dict[str, Any]: timeout=self.timeout, impersonate=self.fingerprint.get("browser_type", "chrome110") ) - + if response.status_code == 200: return response.json() elif response.status_code == 401: @@ -464,7 +462,7 @@ def get_available_models(self) -> Dict[str, Any]: ) else: raise exceptions.FailedToGenerateResponseError(f"Failed to get models: HTTP {response.status_code}") - + except CurlError as e: raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {e}") except Exception as e: @@ -473,6 +471,6 @@ def get_available_models(self) -> Dict[str, Any]: if __name__ == "__main__": ai = QodoAI() # u will need to give your API key here to get api install qodo cli via https://docs.qodo.ai/qodo-documentation/qodo-gen-cli/getting-started/setup-and-quickstart - response = ai.chat("write a poem about india", raw=False, stream=True) + response = ai.chat("write a poem about india", raw=False, stream=True) for chunk in response: - print(chunk, end='', flush=True) \ No newline at end of file + print(chunk, end='', flush=True) diff --git a/webscout/Provider/UNFINISHED/XenAI.py b/webscout/Provider/UNFINISHED/XenAI.py index 87a3192d..dc924821 100644 --- a/webscout/Provider/UNFINISHED/XenAI.py +++ b/webscout/Provider/UNFINISHED/XenAI.py @@ -1,17 +1,15 @@ -import json -import uuid import random import string +import uuid +import warnings from typing import Any, Dict, Generator, Union + import requests -import warnings import urllib3 -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent # Suppress only the single InsecureRequestWarning from urllib3 needed for verify=False @@ -69,7 +67,7 @@ def __init__( print(f"Warning: Model '{model}' is not listed in AVAILABLE_MODELS. Proceeding with the provided model.") self.api_endpoint = "https://chat.xenai.tech/api/chat/completions" - + self.model = model self.system_prompt = system_prompt diff --git a/webscout/Provider/UNFINISHED/Youchat.py b/webscout/Provider/UNFINISHED/Youchat.py index 349decd5..81f25a9d 100644 --- a/webscout/Provider/UNFINISHED/Youchat.py +++ b/webscout/Provider/UNFINISHED/Youchat.py @@ -1,14 +1,14 @@ -from uuid import uuid4 -import json import datetime -from typing import Union, Dict, Any, Generator -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider -from webscout import exceptions +import json +from typing import Generator, Union +from uuid import uuid4 + import cloudscraper +from webscout import exceptions +from webscout.AIbase import Provider +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers + class YouChat(Provider): """ @@ -34,7 +34,7 @@ class YouChat(Provider): # "deepseek_r1", # isProOnly: true # "deepseek_v3", # isProOnly: true # "gemini_2_5_pro_experimental", # isProOnly: true - + # Free models (isProOnly: false) "gpt_4o_mini", "gpt_4o", @@ -52,7 +52,7 @@ class YouChat(Provider): "llama3_1_405b", "mistral_large_2", "command_r_plus", - + # Free models not enabled for user chat modes "llama3_3_70b", # isAllowedForUserChatModes: false "llama3_2_90b", # isAllowedForUserChatModes: false @@ -174,10 +174,10 @@ def ask( trace_id = str(uuid4()) conversation_turn_id = str(uuid4()) - + # Current timestamp in ISO format for traceId current_time = datetime.datetime.now().isoformat() - + # Updated query parameters to match the new API format params = { "page": 1, @@ -197,7 +197,7 @@ def ask( "traceId": f"{trace_id}|{conversation_turn_id}|{current_time}", "use_nested_youchat_updates": "true" } - + # New payload format is JSON payload = { "query": conversation_prompt, @@ -206,12 +206,12 @@ def ask( def for_stream(): response = self.session.post( - self.chat_endpoint, - headers=self.headers, - cookies=self.cookies, + self.chat_endpoint, + headers=self.headers, + cookies=self.cookies, params=params, data=json.dumps(payload), - stream=True, + stream=True, timeout=self.timeout ) if not response.ok: @@ -312,11 +312,11 @@ def get_message(self, response: dict) -> str: print("-" * 80) print(f"{'Model':<50} {'Status':<10} {'Response'}") print("-" * 80) - + # Test all available models working = 0 total = len(YouChat.AVAILABLE_MODELS) - + for model in YouChat.AVAILABLE_MODELS: try: test_ai = YouChat(model=model, timeout=60) @@ -325,7 +325,7 @@ def get_message(self, response: dict) -> str: for chunk in response: response_text += chunk print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True) - + if response_text and len(response_text.strip()) > 0: status = "✓" # Truncate response if too long diff --git a/webscout/Provider/UNFINISHED/aihumanizer.py b/webscout/Provider/UNFINISHED/aihumanizer.py index cfd4ec32..2b7a436c 100644 --- a/webscout/Provider/UNFINISHED/aihumanizer.py +++ b/webscout/Provider/UNFINISHED/aihumanizer.py @@ -1,6 +1,10 @@ -import requests import uuid + +import requests +from rich import print + from webscout.litagent import LitAgent + url = 'https://aihumanizer.work/api/v1/text/rewriter' headers = { @@ -33,5 +37,5 @@ } response = requests.post(url, headers=headers, cookies=cookies, json=json_data) -from rich import print -print(response.json()) \ No newline at end of file + +print(response.json()) diff --git a/webscout/Provider/UNFINISHED/grammerchecker.py b/webscout/Provider/UNFINISHED/grammerchecker.py index 90d6b9cf..21b9edba 100644 --- a/webscout/Provider/UNFINISHED/grammerchecker.py +++ b/webscout/Provider/UNFINISHED/grammerchecker.py @@ -1,5 +1,6 @@ import requests + def create_grammar_check_job(text: str): url = 'https://api.aigrammarchecker.io/api/ai-check-grammar/create-job' headers = { @@ -32,4 +33,4 @@ def create_grammar_check_job(text: str): if __name__ == "__main__": from rich import print as cprint - cprint(create_grammar_check_job("she gg to school")) \ No newline at end of file + cprint(create_grammar_check_job("she gg to school")) diff --git a/webscout/Provider/UNFINISHED/liner.py b/webscout/Provider/UNFINISHED/liner.py index 75a7ec6c..cb4e83fc 100644 --- a/webscout/Provider/UNFINISHED/liner.py +++ b/webscout/Provider/UNFINISHED/liner.py @@ -1,15 +1,20 @@ -import requests import json import time import uuid from copy import deepcopy -from typing import List, Dict, Optional, Union, Generator, Any +from typing import Any, Dict, Generator, List, Optional, Union + +import requests # Import base classes and utility structures -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseChat, BaseCompletions +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( - ChatCompletionChunk, ChatCompletion, Choice, ChoiceDelta, - ChatCompletionMessage, CompletionUsage + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Choice, + ChoiceDelta, + CompletionUsage, ) # Attempt to import LitAgent, fallback if not available @@ -46,7 +51,7 @@ def create( if message.get("role") == "user": user_content = message.get("content") break - + if not user_content: raise ValueError("At least one user message is required") @@ -54,7 +59,7 @@ def create( payload = deepcopy(self._client.base_payload) payload["query"] = user_content payload["modelType"] = model - + request_id = f"chatcmpl-{uuid.uuid4()}" created_time = int(time.time()) @@ -84,21 +89,21 @@ def _create_stream( continue if not line.startswith("data:"): continue - + data_str = line[6:].strip() if not data_str: continue - + try: event = json.loads(data_str) except json.JSONDecodeError: continue - + chunk_content = event.get("answer") if chunk_content: delta = ChoiceDelta(content=chunk_content, role="assistant") choice = Choice(index=0, delta=delta, finish_reason=None) - + chunk = ChatCompletionChunk( id=request_id, object="chat.completion.chunk", @@ -107,7 +112,7 @@ def _create_stream( choices=[choice] ) yield chunk - + # Send final chunk with finish_reason final_delta = ChoiceDelta(content=None) final_choice = Choice(index=0, delta=final_delta, finish_reason="stop") @@ -145,16 +150,16 @@ def _create_non_stream( continue if not line.startswith("data:"): continue - + data_str = line[6:].strip() if not data_str: continue - + try: event = json.loads(data_str) except json.JSONDecodeError: continue - + chunk = event.get("answer") if chunk: answer_parts.append(chunk) @@ -163,7 +168,7 @@ def _create_non_stream( raise IOError("No answer content received from Liner") full_content = "".join(answer_parts) - + message = ChatCompletionMessage(role="assistant", content=full_content) choice = Choice(index=0, message=message, finish_reason="stop") usage = CompletionUsage( @@ -195,7 +200,7 @@ class Liner(OpenAICompatibleProvider): Liner AI provider for OpenAI-compatible API. Supports claude-4-5-sonnet and other models via Liner's search-enhanced AI. """ - + AVAILABLE_MODELS = [ "claude-4-5-sonnet", "gpt-4", @@ -211,7 +216,7 @@ def __init__( ): """ Initialize the Liner provider. - + Args: api_key: Not used, kept for compatibility model: Model to use (default: claude-4-5-sonnet) @@ -225,9 +230,9 @@ def __init__( self.model = model self.timeout = timeout self.proxies = proxies - + self.base_url = "https://getliner.com/lisa/v2/answer?lpv=250414" - + self.headers = { "accept": "text/event-stream", "accept-language": "en-US,en;q=0.9,en-IN;q=0.8", @@ -246,7 +251,7 @@ def __init__( "sentry-trace": "30690146d0014887896af7e513702e97-92aa3c44e2ecea6b-0", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36 Edg/141.0.0.0", } - + self.cookies = { "_k_state": "MmRiY2RjYTgtYjExMi00NTY1LWI1NWQtZjZkZjQ4NjFkZTE3LS0wNzZmNjVmNC05MTIzLTQ1MmItODQzMC0yYmRlYmEyNTA3NjQ=", "__stripe_mid": "b3b3a7d0-d8a7-41b8-a75c-df41869803e0f06952", @@ -269,7 +274,7 @@ def __init__( "_dd_s": "aid=c19eef5b-af5a-4b9a-97b1-5d4ef1bfeff4&rum=0&expire=1760087072218", "AMP_ac91207b66": "JTdCJTIyZGV2aWNlSWQlMjIlM0ElMjJiOTI5MzI0MC0wMTJmLTRlOTctYjUwYi03ZTdiNDIxM2RiZTAlMjIlMkMlMjJ1c2VySWQlMjIlM0ElMjI4NTI2MzE0JTIyJTJDJTIyc2Vzc2lvbklkJTIyJTNBMTc2MDA4NTQzOTg5NiUyQyUyMm9wdE91dCUyMiUzQWZhbHNlJTJDJTIybGFzdEV2ZW50VGltZSUyMiUzQTE3NjAwODYxNzIyMzclMkMlMjJsYXN0RXZlbnRJZCUyMiUzQTk3JTJDJTIycGFnZUNvdW50ZXIlMjIlM0E1JTdE", } - + self.base_payload = { "spaceId": 17788115, "threadId": 89019415, @@ -288,7 +293,7 @@ def __init__( "isDeepResearchMode": False, "answerFormat": "auto", } - + self.session = requests.Session() self.chat = Chat(self) @@ -307,7 +312,7 @@ def list(inner_self): if __name__ == "__main__": # Example usage client = Liner(model="claude-4-5-sonnet") - + # Non-streaming example response = client.chat.completions.create( model="claude-4-5-sonnet", @@ -318,7 +323,7 @@ def list(inner_self): ) print("Non-streaming response:") print(response.choices[0].message.content) - + # Streaming example print("\nStreaming response:") stream = client.chat.completions.create( diff --git a/webscout/Provider/UNFINISHED/liner_api_request.py b/webscout/Provider/UNFINISHED/liner_api_request.py index 88fdfd01..36531969 100644 --- a/webscout/Provider/UNFINISHED/liner_api_request.py +++ b/webscout/Provider/UNFINISHED/liner_api_request.py @@ -1,25 +1,23 @@ -import requests import json -from typing import Dict, Optional, Generator, Union, Any -from uuid import uuid4 -import time -import base64 import random +from typing import Dict, Generator, Optional, Union +from uuid import uuid4 + +import requests +from webscout import LitAgent, exceptions from webscout.AIbase import AISearch -from webscout import exceptions -from webscout import LitAgent class Response: """A wrapper class for Liner API responses. - + This class automatically converts response objects to their text representation when printed or converted to string. - + Attributes: text (str): The text content of the response - + Example: >>> response = Response("Hello, world!") >>> print(response) @@ -29,20 +27,20 @@ class Response: """ def __init__(self, text: str): self.text = text - + def __str__(self): return self.text - + def __repr__(self): return self.text class Liner(AISearch): """A class to interact with the Liner AI search API. - + Liner provides a powerful search interface that returns AI-generated responses based on web content. It supports both streaming and non-streaming responses. - + Basic Usage: >>> from webscout import Liner >>> ai = Liner(cookies_path="cookies.json") @@ -50,18 +48,18 @@ class Liner(AISearch): >>> response = ai.search("What is Python?") >>> print(response) Python is a high-level programming language... - + >>> # Streaming example >>> for chunk in ai.search("Tell me about AI", stream=True): ... print(chunk, end="", flush=True) Artificial Intelligence is... - + >>> # Raw response format >>> for chunk in ai.search("Hello", stream=True, raw=True): ... print(chunk) {'text': 'Hello'} {'text': ' there!'} - + Args: cookies_path (str): Path to the cookies JSON file timeout (int, optional): Request timeout in seconds. Defaults to 30. @@ -79,7 +77,7 @@ def __init__( reasoning_mode: bool = False, ): """Initialize the Liner API client. - + Args: cookies_path (str): Path to the cookies JSON file timeout (int, optional): Request timeout in seconds. Defaults to 30. @@ -95,13 +93,13 @@ def __init__( self.cookies_path = cookies_path self.deep_search = deep_search self.reasoning_mode = reasoning_mode - + # Generate random IDs self.space_id = random.randint(10000000, 99999999) self.thread_id = random.randint(10000000, 99999999) self.user_message_id = random.randint(100000000, 999999999) self.user_id = random.randint(1000000, 9999999) - + self.headers = { "accept": "text/event-stream", "accept-encoding": "gzip, deflate, br, zstd", @@ -119,7 +117,7 @@ def __init__( "sec-gpc": "1", "user-agent": LitAgent().random() } - + # Load cookies from JSON file self.cookies = self._load_cookies() if not self.cookies: @@ -132,7 +130,7 @@ def __init__( def _load_cookies(self) -> Optional[Dict[str, str]]: """Load cookies from a JSON file. - + Returns: Optional[Dict[str, str]]: Dictionary of cookies if successful, None otherwise """ @@ -157,7 +155,7 @@ def search( raw: bool = False, ) -> Union[Response, Generator[Union[Dict[str, str], Response], None, None]]: """Search using the Liner API and get AI-generated responses. - + Args: prompt (str): The search query or prompt to send to the API. stream (bool, optional): If True, yields response chunks as they arrive. @@ -165,12 +163,12 @@ def search( raw (bool, optional): If True, returns raw response dictionaries with 'text' key. If False, returns Response objects that convert to text automatically. Defaults to False. - + Returns: - Union[Response, Generator[Union[Dict[str, str], Response], None, None]]: + Union[Response, Generator[Union[Dict[str, str], Response], None, None]]: - If stream=False: Returns complete response - If stream=True: Yields response chunks as they arrive - + Raises: APIConnectionError: If the API request fails """ @@ -192,7 +190,7 @@ def search( "experimentVariants": [], "isDeepResearchMode": self.deep_search } - + def for_stream(): try: with self.session.post( @@ -205,18 +203,18 @@ def for_stream(): raise exceptions.APIConnectionError( f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}" ) - + current_reasoning = "" current_answer = "" - + for line in response.iter_lines(decode_unicode=True): if line == "event:finish_answer": break - + if line.startswith('data:'): try: data = json.loads(line[5:]) # Remove 'data:' prefix - + # Handle reasoning updates if enabled if self.reasoning_mode and 'reasoning' in data: current_reasoning += data['reasoning'] @@ -224,7 +222,7 @@ def for_stream(): yield {"text": data['reasoning']} else: yield Response(data['reasoning']) - + # Handle answer updates if 'answer' in data: current_answer += data['answer'] @@ -232,13 +230,13 @@ def for_stream(): yield {"text": data['answer']} else: yield Response(data['answer']) - + except json.JSONDecodeError: continue - + except requests.exceptions.RequestException as e: raise exceptions.APIConnectionError(f"Request failed: {e}") - + def for_non_stream(): full_response = "" for chunk in for_stream(): @@ -246,7 +244,7 @@ def for_non_stream(): yield chunk else: full_response += str(chunk) - + if not raw: self.last_response = Response(full_response) return self.last_response @@ -256,8 +254,8 @@ def for_non_stream(): if __name__ == "__main__": from rich import print - + ai = Liner(cookies_path="cookies.json") response = ai.search(input(">>> "), stream=True, raw=False) for chunk in response: - print(chunk, end="", flush=True) \ No newline at end of file + print(chunk, end="", flush=True) diff --git a/webscout/Provider/UNFINISHED/samurai.py b/webscout/Provider/UNFINISHED/samurai.py index 8a9d7cca..a1339c39 100644 --- a/webscout/Provider/UNFINISHED/samurai.py +++ b/webscout/Provider/UNFINISHED/samurai.py @@ -1,12 +1,12 @@ -from typing import * -from webscout.AIutel import Conversation -from webscout.AIutel import Optimizers -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider -from webscout import exceptions -from curl_cffi.requests import Session -from curl_cffi import CurlError import json +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi.requests import Session + +from webscout import exceptions +from webscout.AIbase import Provider +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream + class samurai(Provider): """ @@ -221,4 +221,4 @@ def get_message(self, response: dict) -> str: display_text = "Empty or invalid response" print(f"\r{model:<50} {status:<10} {display_text}") except Exception as e: - print(f"\r{model:<50} {'✗':<10} {str(e)}") \ No newline at end of file + print(f"\r{model:<50} {'✗':<10} {str(e)}") diff --git a/webscout/Provider/Venice.py b/webscout/Provider/Venice.py index 25bfae8a..a1820978 100644 --- a/webscout/Provider/Venice.py +++ b/webscout/Provider/Venice.py @@ -1,29 +1,33 @@ -from curl_cffi import CurlError -from curl_cffi.requests import Session # Import Session -import json -from typing import Generator, Dict, Any, List, Optional, Union -from uuid import uuid4 import random +from typing import Any, Dict, Generator, Optional, Union +from uuid import uuid4 + +from curl_cffi import CurlError +from curl_cffi.requests import Session # Import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) from webscout.litagent import LitAgent + class Venice(Provider): """ A class to interact with the Venice AI API. """ - + required_auth = False AVAILABLE_MODELS = [ "mistral-31-24b", "dolphin-3.0-mistral-24b", "dolphin-3.0-mistral-24b-1dot1" ] - + def __init__( self, is_conversation: bool = True, @@ -31,24 +35,24 @@ def __init__( timeout: int = 30, temperature: float = 0.8, # Keep temperature, user might want to adjust top_p: float = 0.9, # Keep top_p - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, - model: str = "mistral-31-24b", + act: Optional[str] = None, + model: str = "mistral-31-24b", # System prompt is empty in the example, but keep it configurable - system_prompt: str = "" + system_prompt: str = "" ): """Initialize Venice AI client""" if model not in self.AVAILABLE_MODELS: raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}") - + # Update API endpoint - self.api_endpoint = "https://outerface.venice.ai/api/inference/chat" + self.api_endpoint = "https://outerface.venice.ai/api/inference/chat" # Initialize curl_cffi Session - self.session = Session() + self.session = Session() self.is_conversation = is_conversation self.max_tokens_to_sample = max_tokens self.temperature = temperature @@ -57,7 +61,7 @@ def __init__( self.model = model self.system_prompt = system_prompt self.last_response = {} - + # Update Headers based on successful request self.headers = { "User-Agent": LitAgent().random(), # Keep using LitAgent @@ -67,15 +71,15 @@ def __init__( "origin": "https://venice.ai", "referer": "https://venice.ai/", # Update referer # Update sec-ch-ua to match example - "sec-ch-ua": '"Microsoft Edge";v="135", "Not-A.Brand";v="8", "Chromium";v="135"', + "sec-ch-ua": '"Microsoft Edge";v="135", "Not-A.Brand";v="8", "Chromium";v="135"', "sec-ch-ua-mobile": "?0", "sec-ch-ua-platform": '"Windows"', "sec-fetch-dest": "empty", "sec-fetch-mode": "cors", # Update sec-fetch-site to match example - "sec-fetch-site": "same-site", + "sec-fetch-site": "same-site", # Add missing headers from example - "priority": "u=1, i", + "priority": "u=1, i", "sec-gpc": "1", "x-venice-version": "interface@20250424.065523+50bac27" # Add version header } @@ -83,7 +87,7 @@ def __init__( # Update curl_cffi session headers and proxies self.session.headers.update(self.headers) self.session.proxies.update(proxies) - + self.__available_optimizers = ( method for method in dir(Optimizers) @@ -115,9 +119,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: if optimizer in self.__available_optimizers: @@ -145,12 +150,12 @@ def ask( def for_stream(): try: response = self.session.post( - self.api_endpoint, - json=payload, - stream=True, + self.api_endpoint, + json=payload, + stream=True, timeout=self.timeout, impersonate="edge101" - ) + ) if response.status_code != 200: raise exceptions.FailedToGenerateResponseError( f"Request failed with status code {response.status_code} - {response.text}" @@ -175,30 +180,30 @@ def for_stream(): streaming_text += content_chunk yield dict(text=content_chunk) self.conversation.update_chat_history(prompt, streaming_text) - self.last_response = {"text": streaming_text} - except CurlError as e: + self.last_response = {"text": streaming_text} + except CurlError as e: raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {e}") - except Exception as e: + except Exception as e: raise exceptions.FailedToGenerateResponseError(f"An unexpected error occurred ({type(e).__name__}): {e}") def for_non_stream(): full_text = "" - for chunk_data in for_stream(): + for chunk_data in for_stream(): if isinstance(chunk_data, dict) and "text" in chunk_data: full_text += chunk_data["text"] - elif isinstance(chunk_data, str): + elif isinstance(chunk_data, str): full_text += chunk_data - self.last_response = {"text": full_text} - return self.last_response + self.last_response = {"text": full_text} + return self.last_response return for_stream() if stream else for_non_stream() def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, # Added raw parameter - ) -> Union[str, Generator]: + ) -> Union[str, Generator[str, None, None]]: def for_stream(): for response in self.ask(prompt, True, raw=raw, optimizer=optimizer, conversationally=conversationally): if raw: @@ -222,20 +227,23 @@ def get_message(self, response: dict) -> str: print("-" * 80) print(f"{'Model':<50} {'Status':<10} {'Response'}") print("-" * 80) - + # Test all available models working = 0 total = len(Venice.AVAILABLE_MODELS) - + for model in Venice.AVAILABLE_MODELS: try: test_ai = Venice(model=model, timeout=60) response = test_ai.chat("Say 'Hello' in one word", stream=True) response_text = "" - for chunk in response: - response_text += chunk - print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True) - + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + response_text += chunk + print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True) + else: + response_text = str(response) + if response_text and len(response_text.strip()) > 0: status = "✓" # Truncate response if too long @@ -245,4 +253,4 @@ def get_message(self, response: dict) -> str: display_text = "Empty or invalid response" print(f"\r{model:<50} {status:<10} {display_text}") except Exception as e: - print(f"\r{model:<50} {'✗':<10} {str(e)}") \ No newline at end of file + print(f"\r{model:<50} {'✗':<10} {str(e)}") diff --git a/webscout/Provider/VercelAI.py b/webscout/Provider/VercelAI.py index 7786d58e..abc237f4 100644 --- a/webscout/Provider/VercelAI.py +++ b/webscout/Provider/VercelAI.py @@ -1,15 +1,19 @@ +import json import re import time -from curl_cffi import requests -import json -from typing import Union, Any, Dict, Generator, Optional import uuid +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import requests -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) from webscout.litagent import LitAgent @@ -29,12 +33,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 600, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "chat-model", system_prompt: str = "You are a helpful AI assistant." ): @@ -128,9 +132,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator[Any, None, None]]: + **kwargs: Any, + ) -> Response: """Chat with AI""" conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: @@ -194,10 +199,10 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, # Added raw parameter - ) -> str: + ) -> Union[str, Generator[str, None, None]]: def for_stream(): for response in self.ask( prompt, True, raw=raw, optimizer=optimizer, conversationally=conversationally @@ -222,7 +227,7 @@ def for_non_stream(): def get_message(self, response: dict) -> str: """Retrieves message only from response""" - assert isinstance(response, dict), "Response should be of dict data-type only" + assert isinstance(response, dict), "Response should be of dict data-type only" # Formatting is handled by the extractor now text = response.get("text", "") return text.replace('\\n', '\n').replace('\\n\\n', '\n\n') # Keep newline replacement if needed @@ -231,20 +236,23 @@ def get_message(self, response: dict) -> str: print("-" * 80) print(f"{'Model':<50} {'Status':<10} {'Response'}") print("-" * 80) - + # Test all available models working = 0 total = len(VercelAI.AVAILABLE_MODELS) - + for model in VercelAI.AVAILABLE_MODELS: try: test_ai = VercelAI(model=model, timeout=60) response = test_ai.chat("Say 'Hello' in one word", stream=True) response_text = "" - for chunk in response: - response_text += chunk - print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True) - + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + response_text += chunk + print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True) + else: + response_text = str(response) + if response_text and len(response_text.strip()) > 0: status = "✓" # Truncate response if too long @@ -254,4 +262,4 @@ def get_message(self, response: dict) -> str: display_text = "Empty or invalid response" print(f"\r{model:<50} {status:<10} {display_text}") except Exception as e: - print(f"\r{model:<50} {'✗':<10} {str(e)}") \ No newline at end of file + print(f"\r{model:<50} {'✗':<10} {str(e)}") diff --git a/webscout/Provider/WiseCat.py b/webscout/Provider/WiseCat.py index 6ee7e7e5..0130ff2d 100644 --- a/webscout/Provider/WiseCat.py +++ b/webscout/Provider/WiseCat.py @@ -1,14 +1,16 @@ -import re -import json -from typing import Union, Any, Dict, Generator, Optional -from curl_cffi import CurlError +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import CurlError from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) from webscout.litagent import LitAgent @@ -28,12 +30,12 @@ def __init__(self, is_conversation: bool = True, max_tokens: int = 600, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "chat-model-small", system_prompt: str = "You are a helpful AI assistant." ): @@ -48,14 +50,14 @@ def __init__(self, self.max_tokens_to_sample = max_tokens self.api_endpoint = "https://wise-cat-groq.vercel.app/api/chat" # stream_chunk_size is not directly applicable to curl_cffi iter_lines - # self.stream_chunk_size = 64 + # self.stream_chunk_size = 64 self.timeout = timeout self.last_response = {} self.model = model self.system_prompt = system_prompt self.litagent = LitAgent() # Generate headers using LitAgent, but apply them to the curl_cffi session - self.headers = self.litagent.generate_fingerprint() + self.headers = self.litagent.generate_fingerprint() # Update curl_cffi session headers and proxies self.session.headers.update(self.headers) self.session.proxies = proxies @@ -83,9 +85,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator[Any, None, None]]: + **kwargs: Any, + ) -> Response: """Chat with AI""" conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: @@ -114,10 +117,10 @@ def ask( def for_stream(): try: response = self.session.post( - self.api_endpoint, - headers=self.headers, - json=payload, - stream=True, + self.api_endpoint, + headers=self.headers, + json=payload, + stream=True, timeout=self.timeout, impersonate="chrome120" ) @@ -144,7 +147,7 @@ def for_stream(): # Handle unicode escaping and quote unescaping extracted_content = content_chunk.encode().decode('unicode_escape') extracted_content = extracted_content.replace('\\\\', '\\').replace('\\"', '"') - + if raw: yield extracted_content else: @@ -168,10 +171,10 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, # Added raw parameter - ) -> str: + ) -> Union[str, Generator[str, None, None]]: def for_stream(): for response in self.ask( prompt, True, raw=raw, optimizer=optimizer, conversationally=conversationally @@ -206,20 +209,23 @@ def get_message(self, response: dict) -> str: print("-" * 80) print(f"{'Model':<50} {'Status':<10} {'Response'}") print("-" * 80) - + # Test all available models working = 0 total = len(WiseCat.AVAILABLE_MODELS) - + for model in WiseCat.AVAILABLE_MODELS: try: test_ai = WiseCat(model=model, timeout=60) response = test_ai.chat("Say 'Hello' in one word", stream=True) response_text = "" - for chunk in response: - response_text += chunk - print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True) - + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + response_text += chunk + print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True) + else: + response_text = str(response) + if response_text and len(response_text.strip()) > 0: status = "✓" # Truncate response if too long @@ -229,4 +235,4 @@ def get_message(self, response: dict) -> str: display_text = "Empty or invalid response" print(f"\r{model:<50} {status:<10} {display_text}") except Exception as e: - print(f"\r{model:<50} {'✗':<10} {str(e)}") \ No newline at end of file + print(f"\r{model:<50} {'✗':<10} {str(e)}") diff --git a/webscout/Provider/WrDoChat.py b/webscout/Provider/WrDoChat.py index 0cb95930..68401982 100644 --- a/webscout/Provider/WrDoChat.py +++ b/webscout/Provider/WrDoChat.py @@ -1,18 +1,18 @@ import json import re -from typing import Optional, Union, Any, Dict, Generator from datetime import datetime +from typing import Any, Dict, Generator, Optional, Union from uuid import uuid4 + from curl_cffi import CurlError from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent + class WrDoChat(Provider): """ A class to interact with the oi.wr.do chat API. @@ -58,12 +58,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 2000, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "gemini-2.5-flash-preview-04-17", system_prompt: str = "You are a helpful AI assistant.", ): @@ -149,7 +149,7 @@ def _load_cookies(self) -> Optional[Dict[str, str]]: def _wrdo_extractor(self, line: Union[str, Dict[str, Any]]) -> Optional[str]: """Extracts content from the oi.wr.do stream format. - + Format: f:{"messageId":"..."} 0:"content chunk" @@ -163,7 +163,7 @@ def _wrdo_extractor(self, line: Union[str, Dict[str, Any]]) -> Optional[str]: # Decode potential unicode escapes like \u00e9 content = match.group(1).encode().decode('unicode_escape') return content.replace('\\\\', '\\').replace('\\"', '"') # Handle escaped backslashes and quotes - + # Store message ID from 'f:' response elif line.startswith('f:'): try: @@ -188,9 +188,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator[Dict[str, Any], None, None]]: + **kwargs: Any, + ) -> Response: """ Send a message to the oi.wr.do API. @@ -217,7 +218,8 @@ def ask( chat_id = str(uuid4()) message_id = str(uuid4()) - current_time = datetime.utcnow().isoformat() + "Z" + from datetime import timezone + current_time = datetime.now(timezone.utc).isoformat() + "Z" payload = { "id": chat_id, @@ -304,7 +306,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, # Added raw parameter ) -> Union[str, Generator[str, None, None]]: @@ -342,26 +344,31 @@ def for_non_stream(): return self.get_message(result) return for_stream() if stream else for_non_stream() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """ Extract message from response. Args: - response (dict): The response dictionary. + response (Response): The response dictionary. Returns: str: The extracted message. """ - assert isinstance(response, dict), "Response should be of dict data-type only" + if not isinstance(response, dict): + return str(response) return response.get("text", "") if __name__ == "__main__": - from rich import print import json - + + from rich import print + # Example usage ai = WrDoChat(cookies_path="cookies.json") response = ai.chat("write me a poem about AI", stream=True) - for chunk in response: - print(chunk, end="", flush=True) \ No newline at end of file + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end="", flush=True) + else: + print(response) diff --git a/webscout/Provider/__init__.py b/webscout/Provider/__init__.py index 808494f9..d7f812e7 100644 --- a/webscout/Provider/__init__.py +++ b/webscout/Provider/__init__.py @@ -5,6 +5,7 @@ from webscout.Provider.Algion import Algion from webscout.Provider.Andi import AndiSearch from webscout.Provider.Apriel import Apriel +from webscout.Provider.Ayle import Ayle from webscout.Provider.cerebras import Cerebras from webscout.Provider.ChatSandbox import ChatSandbox from webscout.Provider.ClaudeOnline import ClaudeOnline @@ -15,13 +16,12 @@ from webscout.Provider.elmo import Elmo from webscout.Provider.EssentialAI import EssentialAI from webscout.Provider.ExaAI import ExaAI -from webscout.Provider.Ayle import Ayle from webscout.Provider.Gemini import GEMINI from webscout.Provider.geminiapi import GEMINIAPI from webscout.Provider.GithubChat import GithubChat -from webscout.Provider.HadadXYZ import HadadXYZ from webscout.Provider.Gradient import Gradient from webscout.Provider.Groq import GROQ +from webscout.Provider.HadadXYZ import HadadXYZ from webscout.Provider.HeckAI import HeckAI from webscout.Provider.HuggingFace import HuggingFace from webscout.Provider.IBM import IBM @@ -37,7 +37,6 @@ from webscout.Provider.Netwrck import Netwrck from webscout.Provider.Nvidia import Nvidia from webscout.Provider.oivscode import oivscode -from .Openai import OPENAI from webscout.Provider.PI import PiAI from webscout.Provider.QwenLM import QwenLM from webscout.Provider.Sambanova import Sambanova @@ -55,7 +54,8 @@ from webscout.Provider.WiseCat import WiseCat from webscout.Provider.WrDoChat import WrDoChat from webscout.Provider.x0gpt import X0GPT -from webscout.Provider.yep import YEPCHAT + +from .Openai import OPENAI # List of all exported names __all__ = [ @@ -114,5 +114,4 @@ "WiseCat", "WrDoChat", "X0GPT", - "YEPCHAT", ] diff --git a/webscout/Provider/ai4chat.py b/webscout/Provider/ai4chat.py index 105b5559..17ecee01 100644 --- a/webscout/Provider/ai4chat.py +++ b/webscout/Provider/ai4chat.py @@ -1,11 +1,11 @@ -from curl_cffi.requests import Session, RequestsError import urllib.parse -from typing import Union, Any, Dict +from typing import Any, Generator, Optional, Union + +from curl_cffi.requests import RequestsError, Session + +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts -from webscout.AIbase import Provider class AI4Chat(Provider): """ @@ -17,12 +17,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 600, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, system_prompt: str = "You are a helpful and informative AI assistant.", country: str = "Asia", user_id: str = "usersmjb2oaz7y" @@ -65,18 +65,19 @@ def __init__( is_conversation, self.max_tokens_to_sample, filepath, update_file ) self.conversation.history_offset = history_offset - self.system_prompt = system_prompt + self.system_prompt = system_prompt def ask( self, prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - country: str = None, - user_id: str = None, - ): + country: Optional[str] = None, + user_id: Optional[str] = None, + **kwargs: Any, + ) -> Response: """ Sends a prompt to the AI4Chat API and returns the response. If stream=True, yields small chunks of the response (simulated streaming). @@ -127,11 +128,12 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - country: str = None, - user_id: str = None, - ): + country: Optional[str] = None, + user_id: Optional[str] = None, + **kwargs: Any, + ) -> Union[str, Generator[str, None, None]]: """ Generates a response from the AI4Chat API. If stream=True, yields each chunk as a string. @@ -168,7 +170,10 @@ def get_message(self, response: Union[dict, str]) -> str: if __name__ == "__main__": from rich import print - ai = AI4Chat() + ai = AI4Chat() response = ai.chat("Tell me about humans in points", stream=True) - for c in response: - print(c, end="") + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for c in response: + print(c, end="") + else: + print(response) diff --git a/webscout/Provider/akashgpt.py b/webscout/Provider/akashgpt.py index a6576c1a..1f4e8f57 100644 --- a/webscout/Provider/akashgpt.py +++ b/webscout/Provider/akashgpt.py @@ -1,17 +1,21 @@ -from typing import Optional, Union, Any, Dict, Generator -from uuid import uuid4 -import cloudscraper import re -import json import time +from typing import Any, Dict, Generator, Optional, Union +from uuid import uuid4 + +import cloudscraper -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) from webscout.litagent import LitAgent + class AkashGPT(Provider): """ A class to interact with the Akash Network Chat API. @@ -40,12 +44,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 600, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, system_prompt: str = "You are a helpful assistant.", model: str = "meta-llama-3-3-70b-instruct", temperature: float = 0.6, @@ -73,7 +77,7 @@ def __init__( # Validate model choice if model not in self.AVAILABLE_MODELS: raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}") - + self.session = cloudscraper.create_scraper() self.is_conversation = is_conversation self.max_tokens_to_sample = max_tokens @@ -113,7 +117,7 @@ def __init__( "sec-fetch-mode": "cors", "sec-fetch-site": "same-origin", "sec-gpc": "1", - "user-agent": self.agent.random() + "user-agent": self.agent.random() } # Set cookies with the session token @@ -154,9 +158,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Dict[str, Any]: + **kwargs: Any, + ) -> Response: """ Sends a prompt to the Akash Network API and returns the response. @@ -250,9 +255,10 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> str: + **kwargs: Any, + ) -> Union[str, Generator[str, None, None]]: """ Generates a response from the AkashGPT API. @@ -319,8 +325,12 @@ def get_message(self, response: dict) -> str: try: test_ai = AkashGPT(model=model, timeout=60, api_key="5ef9b0782df982fab720810f6ee72a9af01ebadbd9eb05adae0ecc8711ec79c5; _ga_LFRGN2J2RV=GS2.1.s1763554272$o4$g1$t1763554284$j48$l0$h0") # Example key response = test_ai.chat("Say 'Hello' in one word") - response_text = response - + + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + response_text = "".join(list(response)) + else: + response_text = str(response) + if response_text and len(response_text.strip()) > 0: status = "✓" # Truncate response if too long @@ -330,4 +340,4 @@ def get_message(self, response: dict) -> str: display_text = "Empty or invalid response" print(f"{model:<50} {status:<10} {display_text}") except Exception as e: - print(f"{model:<50} {'✗':<10} {str(e)}") \ No newline at end of file + print(f"{model:<50} {'✗':<10} {str(e)}") diff --git a/webscout/Provider/cerebras.py b/webscout/Provider/cerebras.py index 15a8368e..b27dbb00 100644 --- a/webscout/Provider/cerebras.py +++ b/webscout/Provider/cerebras.py @@ -1,6 +1,5 @@ import re - # Import trio before curl_cffi to prevent eventlet socket monkey-patching conflicts # See: https://github.com/python-trio/trio/issues/3015 try: @@ -14,7 +13,7 @@ from curl_cffi.requests import Session from webscout import exceptions -from webscout.AIbase import Provider +from webscout.AIbase import Provider, Response from webscout.AIutel import ( # Import sanitize_stream AwesomePrompts, Conversation, @@ -42,18 +41,18 @@ class Cerebras(Provider): ] @classmethod - def get_models(cls, api_key: str = None): + def get_models(cls, api_key: Optional[str] = None): """Fetch available models from Cerebras API. - + Args: api_key (str, optional): Cerebras API key. If not provided, returns default models. - + Returns: list: List of available model IDs """ if not api_key: raise Exception("API key required to fetch models") - + try: # Use a temporary curl_cffi session for this class method temp_session = Session() @@ -61,37 +60,37 @@ def get_models(cls, api_key: str = None): "Content-Type": "application/json", "Authorization": f"Bearer {api_key}", } - + response = temp_session.get( "https://api.cerebras.ai/v1/models", headers=headers, impersonate="chrome120" ) - + if response.status_code != 200: raise Exception(f"Failed to fetch models: HTTP {response.status_code}") - + data = response.json() if "data" in data and isinstance(data["data"], list): return [model['id'] for model in data['data']] raise Exception("Invalid response format from API") - + except (curl_cffi.CurlError, Exception) as e: raise Exception(f"Failed to fetch models: {str(e)}") def __init__( self, - cookie_path: str = None, + cookie_path: Optional[str] = None, is_conversation: bool = True, max_tokens: int = 40000, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, - api_key: str = None, + act: Optional[str] = None, + api_key: Optional[str] = None, model: str = "qwen-3-coder-480b", system_prompt: str = "You are a helpful assistant.", temperature: float = 0.7, @@ -296,9 +295,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, # Add raw parameter for consistency - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict, Generator]: + **kwargs: Any, + ) -> Response: """Send a prompt to the model and get a response.""" conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: @@ -341,10 +341,11 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, - ) -> Union[str, Generator]: + **kwargs: Any, + ) -> Union[str, Generator[str, None, None]]: """Chat with the model.""" # Ask returns a generator for stream=True, dict/str for stream=False response_gen_or_dict = self.ask(prompt, stream, raw=raw, optimizer=optimizer, conversationally=conversationally) @@ -364,10 +365,11 @@ def stream_wrapper(): return response_gen_or_dict return self.get_message(response_gen_or_dict) - def get_message(self, response: str) -> str: + def get_message(self, response: Response) -> str: """Retrieves message from response.""" # Updated to handle dict input from ask() - assert isinstance(response, dict), "Response should be of dict data-type only for get_message" + if not isinstance(response, dict): + return str(response) return response.get("text", "") if __name__ == "__main__": diff --git a/webscout/Provider/cleeai.py b/webscout/Provider/cleeai.py index 4bc41459..de0e0ebe 100644 --- a/webscout/Provider/cleeai.py +++ b/webscout/Provider/cleeai.py @@ -1,14 +1,14 @@ -import requests import json +from typing import Any, Generator, Optional, Union from uuid import uuid4 -from typing import Union, Dict, Any, Generator -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider -from webscout import exceptions +import requests + import webscout +from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream + class Cleeai(Provider): """ @@ -20,12 +20,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 600, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, ) -> None: """Instantiates Cleeai @@ -89,9 +89,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> dict: + **kwargs: Any, + ) -> Response: """Chat with AI Args: @@ -178,7 +179,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, ) -> Union[str, Generator[str, None, None]]: @@ -233,5 +234,8 @@ def get_message(self, response: dict) -> str: from rich import print ai = Cleeai(timeout=5000) response = ai.chat("tell me about Abhay koul, HelpingAI", stream=True) - for chunk in response: - print(chunk, end="", flush=True) \ No newline at end of file + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end="", flush=True) + else: + print(response) diff --git a/webscout/Provider/elmo.py b/webscout/Provider/elmo.py index 8f00d16f..d5ca000d 100644 --- a/webscout/Provider/elmo.py +++ b/webscout/Provider/elmo.py @@ -1,14 +1,17 @@ -from curl_cffi.requests import Session +import re # Import re for the extractor +from typing import Any, Dict, Generator, Optional, Union + from curl_cffi import CurlError -import json -from typing import Optional, Union, Any, Dict, Generator +from curl_cffi.requests import Session + from webscout import exceptions -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation, sanitize_stream # Import sanitize_stream -from webscout.AIutel import AwesomePrompts -from webscout.AIbase import Provider -from webscout.litagent import LitAgent -import re # Import re for the extractor +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) class Elmo(Provider): @@ -21,12 +24,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 600, # Note: max_tokens is not used by this API timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, system_prompt: str = "You are a helpful AI assistant. Provide clear, concise, and well-structured information. Organize your responses into paragraphs for better readability.", ) -> None: """Instantiates Elmo @@ -99,11 +102,12 @@ def _elmo_extractor(chunk: Union[str, Dict[str, Any]]) -> Optional[str]: def ask( self, prompt: str, - stream: bool = False, # API supports streaming + stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator[Any, None, None]]: # Corrected return type hint + **kwargs: Any, + ) -> Response: """Chat with AI Args: @@ -230,9 +234,10 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[str, Generator[str, None, None]]: # Corrected return type hint + **kwargs: Any, + ) -> Union[str, Generator[str, None, None]]: """Generate response `str` Args: prompt (str): Prompt to be send. @@ -283,5 +288,8 @@ def get_message(self, response: dict) -> str: from rich import print ai = Elmo() response = ai.chat("write a poem about AI", stream=True) - for chunk in response: - print(chunk, end="", flush=True) \ No newline at end of file + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end="", flush=True) + else: + print(response) diff --git a/webscout/Provider/geminiapi.py b/webscout/Provider/geminiapi.py index 80aa18e9..9e9082be 100644 --- a/webscout/Provider/geminiapi.py +++ b/webscout/Provider/geminiapi.py @@ -5,19 +5,17 @@ """ import json -import os -from typing import Any, Dict, Optional, Generator, Union, List +from typing import Any, Dict, Generator, Optional, Union -from curl_cffi.requests import Session from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent + class GEMINIAPI(Provider): """ A class to interact with the Gemini API using OpenAI-compatible endpoint with LitAgent user-agent. @@ -25,12 +23,12 @@ class GEMINIAPI(Provider): required_auth = True @classmethod - def get_models(cls, api_key: str = None): + def get_models(cls, api_key: Optional[str] = None): """Fetch available models from Gemini API. - + Args: api_key (str, optional): Gemini API key - + Returns: list: List of available model IDs """ @@ -42,21 +40,21 @@ def get_models(cls, api_key: str = None): headers = { "Authorization": f"Bearer {api_key}", } - + response = temp_session.get( "https://generativelanguage.googleapis.com/v1beta/openai/models", headers=headers, impersonate="chrome110" ) - + if response.status_code != 200: raise Exception(f"API request failed with status {response.status_code}: {response.text}") - + data = response.json() if "data" in data and isinstance(data["data"], list): return [model["id"] for model in data["data"] if "id" in model] raise Exception("Invalid response format from API") - + except (CurlError, Exception) as e: raise Exception(f"Failed to fetch models: {str(e)}") @@ -78,12 +76,12 @@ def __init__( top_p: float = 1, model: str = "gemini-1.5-flash", timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, base_url: str = "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", system_prompt: str = "You are a helpful assistant.", browser: str = "chrome" @@ -173,9 +171,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: if optimizer in self.__available_optimizers: @@ -285,8 +284,9 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, + **kwargs: Any, ) -> Union[str, Generator[str, None, None]]: def for_stream_chat(): gen = self.ask( diff --git a/webscout/Provider/julius.py b/webscout/Provider/julius.py index 5a627374..1facf8ad 100644 --- a/webscout/Provider/julius.py +++ b/webscout/Provider/julius.py @@ -1,14 +1,13 @@ +import json import uuid +from typing import Any, Generator, Optional, Union import requests -import json -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider + from webscout import exceptions -from typing import Union, Any, Generator, Dict +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream class Julius(Provider): @@ -37,12 +36,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 600, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "Gemini Flash", ): """Instantiates Julius @@ -57,7 +56,7 @@ def __init__( proxies (dict, optional): Http request proxies. Defaults to {}. history_offset (int, optional): Limit conversation history to this number of last texts. Defaults to 10250. act (str|int, optional): Awesome prompt key or index. (Used as intro). Defaults to None. - model (str, optional): Model to use for generating text. Defaults to "Gemini Flash". + model (str, optional): Model to use for generating text. Defaults to "Gemini Flash". Options: "Llama 3", "GPT-4o", "GPT-3.5", "Command R", "Gemini Flash", "Gemini 1.5". """ if model not in self.AVAILABLE_MODELS: @@ -108,9 +107,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> dict: + **kwargs: Any, + ) -> Response: """Chat with AI Args: @@ -133,7 +133,7 @@ def ask( raise Exception( f"Optimizer is not one of {self.__available_optimizers}" ) - + payload = { "message": {"content": conversation_prompt, "role": "user"}, "provider": "default", @@ -190,7 +190,7 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, ) -> Union[str, Generator[str, None, None]]: @@ -235,5 +235,8 @@ def get_message(self, response: dict) -> str: from rich import print ai = Julius(api_key="",timeout=5000) response = ai.chat("write a poem about AI", stream=True) - for chunk in response: - print(chunk, end="", flush=True) \ No newline at end of file + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end="", flush=True) + else: + print(response) diff --git a/webscout/Provider/learnfastai.py b/webscout/Provider/learnfastai.py index da03473b..5eb9f8dd 100644 --- a/webscout/Provider/learnfastai.py +++ b/webscout/Provider/learnfastai.py @@ -1,15 +1,18 @@ -import os import json -from typing import Any, Dict, Optional, Union, Generator +import os import uuid -from curl_cffi.requests import Session +from typing import Any, Dict, Generator, Optional, Union + from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation, sanitize_stream # Import sanitize_stream -from webscout.AIutel import AwesomePrompts -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, +) class LearnFast(Provider): @@ -22,12 +25,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 600, # Note: max_tokens is not used by this API timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, system_prompt: str = "You are a helpful AI assistant.", # Note: system_prompt is not used by this API ): """ @@ -105,10 +108,10 @@ def upload_image_to_0x0(self, image_path: str) -> str: files = {"file": img_file} try: response = self.session.post( - "https://0x0.st", + "https://0x0.st", files=files, # Add impersonate if using the main session - impersonate="chrome110" + impersonate="chrome110" ) response.raise_for_status() image_url = response.text.strip() @@ -148,12 +151,13 @@ def create_payload( def ask( self, prompt: str, - stream: bool = False, # API supports streaming + stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, image_path: Optional[str] = None, - ) -> Union[dict, Generator[dict, None, None], str]: + **kwargs: Any, + ) -> Response: """Chat with LearnFast Args: @@ -205,10 +209,10 @@ def for_stream(): full_response = "" try: response = self.session.post( - self.api_endpoint, + self.api_endpoint, headers=current_headers, # Use headers with uniqueid - data=data, - stream=True, + data=data, + stream=True, timeout=self.timeout, impersonate="chrome110" ) @@ -261,16 +265,17 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, image_path: Optional[str] = None, - raw: bool = False + raw: bool = False, + **kwargs: Any, ) -> Union[str, Generator[str, None, None]]: """Generate response `str` or stream, with raw support""" try: response_gen = self.ask( prompt, stream=stream, raw=raw, - optimizer=optimizer, conversationally=conversationally, + optimizer=optimizer, conversationally=conversationally, image_path=image_path ) if stream: @@ -306,5 +311,8 @@ def get_message(self, response: dict) -> str: from rich import print ai = LearnFast() response = ai.chat("Hello, how are you?", stream=True, raw=False) - for chunk in response: - print(chunk, end='', flush=True) \ No newline at end of file + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end='', flush=True) + else: + print(response) diff --git a/webscout/Provider/llama3mitril.py b/webscout/Provider/llama3mitril.py index 99372bc4..06c09f0c 100644 --- a/webscout/Provider/llama3mitril.py +++ b/webscout/Provider/llama3mitril.py @@ -1,12 +1,12 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError import json -from typing import Union, Any, Dict, Generator -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts -from webscout.AIbase import Provider +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session + from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers class Llama3Mitril(Provider): @@ -19,12 +19,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 2048, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, system_prompt: str = "You are a helpful, respectful and honest assistant.", temperature: float = 0.8, ): @@ -76,9 +76,10 @@ def ask( prompt: str, stream: bool = True, # API supports streaming raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator[Any, None, None]]: + **kwargs: Any, + ) -> Response: """Sends a prompt to the Llama3 Mitril API and returns the response.""" conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: @@ -129,7 +130,7 @@ def for_stream(): resp = {"text": token_text} # Yield dict or raw string chunk yield resp if not raw else token_text - except (json.JSONDecodeError, IndexError, UnicodeDecodeError) as e: + except (json.JSONDecodeError, IndexError, UnicodeDecodeError): # Ignore errors in parsing specific lines continue @@ -171,8 +172,9 @@ def chat( self, prompt: str, stream: bool = True, # Default to True as API supports it - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, + **kwargs: Any, ) -> Union[str, Generator[str, None, None]]: """Generates a response from the Llama3 Mitril API.""" @@ -212,4 +214,4 @@ def get_message(self, response: Dict[str, Any]) -> str: ) for response in ai.chat("Hello", stream=True): - print(response, end="", flush=True) \ No newline at end of file + print(response, end="", flush=True) diff --git a/webscout/Provider/llmchat.py b/webscout/Provider/llmchat.py index 979c4fdb..febe51c6 100644 --- a/webscout/Provider/llmchat.py +++ b/webscout/Provider/llmchat.py @@ -1,14 +1,12 @@ -from curl_cffi.requests import Session +from typing import Any, Dict, Generator, Union + from curl_cffi import CurlError -import json -from typing import Union, Any, Dict, Optional, Generator, List +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers, sanitize_stream -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts -from webscout.AIbase import Provider from webscout import exceptions -from webscout.litagent import LitAgent as Lit +from webscout.AIbase import Provider +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream + class LLMChat(Provider): """ @@ -95,7 +93,7 @@ def __init__( self.last_response = {} self.model = model self.system_prompt = system_prompt - + self.headers = { "Content-Type": "application/json", "Accept": "*/*", @@ -161,9 +159,9 @@ def for_stream(): full_response = "" try: response = self.session.post( - url, - json=payload, - stream=True, + url, + json=payload, + stream=True, timeout=self.timeout, impersonate="chrome110" ) @@ -252,11 +250,11 @@ def get_message(self, response: Dict[str, Any]) -> str: print("-" * 80) print(f"{'Model':<50} {'Status':<10} {'Response'}") print("-" * 80) - + # Test all available models working = 0 total = len(LLMChat.AVAILABLE_MODELS) - + for model in LLMChat.AVAILABLE_MODELS: try: test_ai = LLMChat(model=model, timeout=60) @@ -265,7 +263,7 @@ def get_message(self, response: Dict[str, Any]) -> str: for chunk in response: response_text += chunk print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True) - + if response_text and len(response_text.strip()) > 0: status = "✓" # Truncate response if too long @@ -279,4 +277,4 @@ def get_message(self, response: Dict[str, Any]) -> str: # ai = LLMChat(model="@cf/meta/llama-3.1-70b-instruct") # response = ai.chat("Say 'Hello' in one word", stream=True, raw=False) # for chunk in response: - # print(chunk, end="", flush=True) \ No newline at end of file + # print(chunk, end="", flush=True) diff --git a/webscout/Provider/llmchatco.py b/webscout/Provider/llmchatco.py index 6dce6e65..1b3a33a1 100644 --- a/webscout/Provider/llmchatco.py +++ b/webscout/Provider/llmchatco.py @@ -1,17 +1,20 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError -import json import uuid -import re -from typing import Union, Any, Dict, Optional, Generator, List +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers, sanitize_stream # Import sanitize_stream -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) from webscout.litagent import LitAgent as Lit + class LLMChatCo(Provider): """ A class to interact with the LLMChat.co API @@ -288,4 +291,4 @@ def get_message(self, response: Dict[str, Any]) -> str: ai = LLMChatCo() response = ai.chat("yooo", stream=True, raw=False) for chunk in response: - print(chunk, end="", flush=True) \ No newline at end of file + print(chunk, end="", flush=True) diff --git a/webscout/Provider/meta.py b/webscout/Provider/meta.py index 1b33f125..e27887e5 100644 --- a/webscout/Provider/meta.py +++ b/webscout/Provider/meta.py @@ -1,22 +1,20 @@ import json +import random import time import urllib import uuid from typing import Dict, Generator, List, Union -import random from curl_cffi import CurlError from curl_cffi.requests import Session -from webscout.scout import Scout +from litprinter import ic -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts -from webscout.AIutel import retry -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, retry from webscout.litagent import LitAgent as Lit -from litprinter import ic +from webscout.scout import Scout + MAX_RETRIES = 3 HTTP2_STREAM_ERRORS = [92, 18, 7, 35, 36] # Common curl HTTP/2 stream errors @@ -266,7 +264,8 @@ def get_fb_session(email, password, proxies=None): "Was not able to login to Facebook. Please check your credentials. " "You may also have been rate limited. Try to connect to Facebook manually." ) - ic.configureOutput(prefix='INFO| '); ic("Successfully logged in to Facebook.") + ic.configureOutput(prefix='INFO| ') + ic("Successfully logged in to Facebook.") return cookies @@ -348,7 +347,7 @@ def __init__( "user-agent": Lit().random(), } ) - + # Configure session for better HTTP/2 handling self.session.timeout = timeout self.session.curl_options.update({ @@ -364,18 +363,18 @@ def __init__( 64: 0, # CURLOPT_SSL_VERIFYPEER 81: 0, # CURLOPT_SSL_VERIFYHOST }) - + # Create a backup session for fallback self.backup_session = Session() self.backup_session.headers.update({"user-agent": Lit().random()}) self.backup_session.curl_options.update(self.session.curl_options) - + # Add HTTP/2 error tracking self.http2_error_count = 0 self.max_http2_errors = 3 self.last_successful_request = time.time() self.retry_count = 0 - + self.access_token = None self.fb_email = fb_email self.fb_password = fb_password @@ -414,7 +413,7 @@ def __init__( self.session.proxies = proxies # If skip_init was True we won't have cookies yet — some methods will fetch them lazily if self.skip_init: - ic.configureOutput(prefix='WARNING| '); + ic.configureOutput(prefix='WARNING| ') ic('Meta initialized in skip_init mode: cookies not fetched. Some operations will fail until cookies are obtained.') @@ -486,7 +485,7 @@ def get_access_token(self) -> str: # Some Curl errors are wrapped in requests.HTTPError from curl_cffi; inspect message err_str = str(e) if 'HTTP/2 stream' in err_str or 'stream' in err_str: - ic.configureOutput(prefix='WARNING| '); + ic.configureOutput(prefix='WARNING| ') ic(f"Detected HTTP/2 stream issue when getting access token: {err_str}. Attempting HTTP/1.1 fallback via requests") try: import requests @@ -615,7 +614,7 @@ def for_stream(): except CurlError as e: # Try HTTP/1.1 fallback once if hasattr(e, 'errno') and e.errno in HTTP2_STREAM_ERRORS: - ic.configureOutput(prefix='WARNING| '); + ic.configureOutput(prefix='WARNING| ') ic("HTTP/2 stream error on streaming request, attempting HTTP/1.1 fallback") try: self.session.curl_options.update({84: 1}) # force HTTP/1.1 @@ -654,7 +653,7 @@ def for_stream(): except CurlError as e: # Handle HTTP/2 stream closure during iteration if hasattr(e, 'errno') and e.errno in HTTP2_STREAM_ERRORS: - ic.configureOutput(prefix='WARNING| '); + ic.configureOutput(prefix='WARNING| ') ic(f"HTTP/2 stream closed during iteration (errno: {e.errno})") if final_message: # Yield the last complete message before the stream closed @@ -663,7 +662,7 @@ def for_stream(): else: raise except (ConnectionError, TimeoutError) as e: - ic.configureOutput(prefix='WARNING| '); + ic.configureOutput(prefix='WARNING| ') ic(f"Connection error during streaming: {e}") if final_message: # Yield the last complete message before the connection was lost @@ -672,12 +671,12 @@ def for_stream(): prompt, self.get_message(self.last_response) ) return - + if final_message: self.conversation.update_chat_history( prompt, self.get_message(self.last_response) ) - + except CurlError as e: if hasattr(e, 'errno') and e.errno in HTTP2_STREAM_ERRORS: raise exceptions.FailedToGenerateResponseError( @@ -697,7 +696,7 @@ def for_stream(): except CurlError as e: # Try HTTP/1.1 fallback for non-stream requests if hasattr(e, 'errno') and e.errno in HTTP2_STREAM_ERRORS: - ic.configureOutput(prefix='WARNING| '); + ic.configureOutput(prefix='WARNING| ') ic("HTTP/2 error on non-stream request, attempting HTTP/1.1 fallback") try: self.session.curl_options.update({84: 1}) # force HTTP/1.1 @@ -873,7 +872,7 @@ def get_cookies(self) -> dict: last_response = response break except Exception as e: - ic.configureOutput(prefix='WARNING| '); + ic.configureOutput(prefix='WARNING| ') ic(f"Attempt {attempt+1} to fetch meta.ai failed: {e}. Retrying...") time.sleep(1 * (2 ** attempt)) if last_response is None: @@ -983,10 +982,10 @@ def get_message(self, response: dict) -> str: for chunk in ai: print(chunk, end="", flush=True) except exceptions.FailedToGenerateResponseError as e: - ic.configureOutput(prefix='ERROR| '); + ic.configureOutput(prefix='ERROR| ') ic(f"Meta provider failed to initialize or run: {e}") ic("Possible causes: network connectivity issues, region blocking, or site returning error pages.") ic("For offline testing, re-run with: Meta(skip_init=True)") except Exception as e: - ic.configureOutput(prefix='ERROR| '); + ic.configureOutput(prefix='ERROR| ') ic(f"Unexpected error running meta provider: {e}") diff --git a/webscout/Provider/oivscode.py b/webscout/Provider/oivscode.py index 7ba3533f..7fcfdaf9 100644 --- a/webscout/Provider/oivscode.py +++ b/webscout/Provider/oivscode.py @@ -1,10 +1,11 @@ import random -import requests import secrets import string import uuid from typing import Any, Dict, Optional, Union +import requests + from webscout import exceptions from webscout.AIbase import Provider from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream diff --git a/webscout/Provider/searchchat.py b/webscout/Provider/searchchat.py index e0b5bad5..5cd90214 100644 --- a/webscout/Provider/searchchat.py +++ b/webscout/Provider/searchchat.py @@ -1,16 +1,20 @@ -from curl_cffi.requests import Session +from datetime import datetime, timezone +from typing import Any, Dict, Generator, Optional, Union + from curl_cffi import CurlError -import json -from datetime import datetime -from typing import Any, Dict, Optional, Generator, Union +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) from webscout.litagent import LitAgent + class SearchChatAI(Provider): """ A class to interact with the SearchChatAI API. @@ -22,12 +26,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 2049, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, ): """Initializes the SearchChatAI API client.""" self.url = "https://search-chat.ai/api/chat-test-stop.php" @@ -35,12 +39,12 @@ def __init__( self.is_conversation = is_conversation self.max_tokens_to_sample = max_tokens self.last_response = {} - + # Initialize LitAgent for user agent generation self.agent = LitAgent() # Use fingerprinting to create a consistent browser identity self.fingerprint = self.agent.generate_fingerprint("chrome") - + # Use the fingerprint for headers self.headers = { "Accept": self.fingerprint["accept"], @@ -57,7 +61,7 @@ def __init__( "Sec-CH-UA-Platform": f'"{self.fingerprint["platform"]}"', "User-Agent": self.fingerprint["user_agent"], } - + # Initialize curl_cffi Session self.session = Session() # Update curl_cffi session headers and proxies @@ -85,13 +89,13 @@ def __init__( def refresh_identity(self, browser: str = None): """ Refreshes the browser identity fingerprint. - + Args: browser: Specific browser to use for the new fingerprint """ browser = browser or self.fingerprint.get("browser_type", "chrome") self.fingerprint = self.agent.generate_fingerprint(browser) - + # Update headers with new fingerprint self.headers.update({ "Accept": self.fingerprint["accept"], @@ -100,11 +104,11 @@ def refresh_identity(self, browser: str = None): "Sec-CH-UA-Platform": f'"{self.fingerprint["platform"]}"', "User-Agent": self.fingerprint["user_agent"], }) - + # Update session headers (already done in the original code, should work with curl_cffi session) for header, value in self.headers.items(): self.session.headers[header] = value - + return self.fingerprint def ask( @@ -112,19 +116,20 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: """ Send a message to the API and get the response. - + Args: prompt: The message to send stream: Whether to stream the response raw: Whether to return raw response optimizer: The optimizer to use conversationally: Whether to use conversation history - + Returns: Either a dictionary with the response or a generator for streaming """ @@ -147,7 +152,7 @@ def ask( "text": conversation_prompt } ], - "timestamp": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") + "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") } ] } @@ -182,7 +187,7 @@ def for_stream(): raise exceptions.FailedToGenerateResponseError( f"Request failed with status code {response.status_code} - {response.text}" ) - + streaming_text = "" # Use sanitize_stream processed_stream = sanitize_stream( @@ -197,12 +202,12 @@ def for_stream(): for content_chunk in processed_stream: if raw: - yield content_chunk + yield content_chunk else: if content_chunk and isinstance(content_chunk, str): streaming_text += content_chunk yield dict(text=content_chunk) - + # Update history and last response after stream finishes self.last_response = {"text": streaming_text} self.conversation.update_chat_history(prompt, streaming_text) @@ -216,13 +221,13 @@ def for_non_stream(): full_text = "" # Iterate through the generator provided by for_stream # Ensure raw=False so for_stream yields dicts - for chunk_data in for_stream(): + for chunk_data in for_stream(): if isinstance(chunk_data, dict) and "text" in chunk_data: full_text += chunk_data["text"] # If raw=True was somehow passed, handle string chunks elif isinstance(chunk_data, str): full_text += chunk_data - + # last_response and history are updated within for_stream # Return the final aggregated response dict or raw string return full_text if raw else self.last_response @@ -234,19 +239,20 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, # Added raw parameter + **kwargs: Any, ) -> Union[str, Generator[str, None, None]]: """ Chat with the API. - + Args: prompt: The message to send stream: Whether to stream the response optimizer: The optimizer to use conversationally: Whether to use conversation history - + Returns: Either a string response or a generator for streaming """ @@ -268,13 +274,17 @@ def for_non_stream_chat(): return self.get_message(response_data) return for_stream_chat() if stream else for_non_stream_chat() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """Extract the message from the response.""" - assert isinstance(response, dict), "Response should be of dict data-type only" - return response["text"] + if not isinstance(response, dict): + return str(response) + return response.get("text", "") if __name__ == "__main__": ai = SearchChatAI() resp = ai.chat("Hello", stream=True, raw=True) - for chunk in resp: - print(chunk, end="") \ No newline at end of file + if hasattr(resp, "__iter__") and not isinstance(resp, (str, bytes)): + for chunk in resp: + print(chunk, end="") + else: + print(resp) diff --git a/webscout/Provider/sonus.py b/webscout/Provider/sonus.py index 7cfeedeb..fc783d94 100644 --- a/webscout/Provider/sonus.py +++ b/webscout/Provider/sonus.py @@ -1,13 +1,19 @@ -from curl_cffi.requests import Session -from curl_cffi import CurlError import json -from typing import Any, Dict, Optional, Generator, Union -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session + from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, +) from webscout.litagent import LitAgent + + class SonusAI(Provider): """ A class to interact with the Sonus AI chat API. @@ -24,20 +30,20 @@ def __init__( is_conversation: bool = True, max_tokens: int = 2049, # Note: max_tokens is not directly used by this API timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "pro" ): """Initializes the Sonus AI API client.""" if model not in self.AVAILABLE_MODELS: raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}") - + self.url = "https://chat.sonus.ai/chat.php" - + # Headers for the request self.headers = { 'Accept': '*/*', @@ -47,7 +53,7 @@ def __init__( 'User-Agent': LitAgent().random() # Add sec-ch-ua headers if needed for impersonation consistency } - + # Initialize curl_cffi Session self.session = Session() # Update curl_cffi session headers and proxies @@ -90,10 +96,11 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, reasoning: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: if optimizer in self.__available_optimizers: @@ -119,9 +126,9 @@ def for_stream(): try: # Use curl_cffi session post with impersonate response = self.session.post( - self.url, + self.url, data=form_data, - stream=True, + stream=True, timeout=self.timeout, impersonate="chrome110" ) @@ -151,11 +158,11 @@ def for_stream(): yield dict(text=content) except json.JSONDecodeError: continue - + # Update history and last response after stream finishes self.last_response = {"text": streaming_text} self.conversation.update_chat_history(prompt, streaming_text) - + except CurlError as e: raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {str(e)}") from e except Exception as e: @@ -165,7 +172,7 @@ def for_non_stream(): try: # Use curl_cffi session post with impersonate response = self.session.post( - self.url, + self.url, # headers are set on the session data=form_data, # Use data for multipart form fields timeout=self.timeout, @@ -196,7 +203,7 @@ def for_non_stream(): self.conversation.update_chat_history(prompt, full_response) # Return dict or raw string return full_response if raw else {"text": full_response} - + except CurlError as e: # Catch CurlError raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {str(e)}") from e except Exception as e: # Catch other potential exceptions @@ -208,10 +215,11 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, reasoning: bool = False, raw: bool = False, # Added raw parameter + **kwargs: Any, ) -> Union[str, Generator[str, None, None]]: def for_stream_chat(): for response in self.ask( @@ -231,12 +239,16 @@ def for_non_stream_chat(): return self.get_message(response_data) return for_stream_chat() if stream else for_non_stream_chat() - def get_message(self, response: dict) -> str: - assert isinstance(response, dict), "Response should be of dict data-type only" - return response["text"] + def get_message(self, response: Response) -> str: + if not isinstance(response, dict): + return str(response) + return response.get("text", "") if __name__ == "__main__": sonus = SonusAI() resp = sonus.chat("Hello", stream=True, raw=True) - for chunk in resp: - print(chunk, end="") \ No newline at end of file + if hasattr(resp, "__iter__") and not isinstance(resp, (str, bytes)): + for chunk in resp: + print(chunk, end="") + else: + print(resp) diff --git a/webscout/Provider/toolbaz.py b/webscout/Provider/toolbaz.py index 5ff5ef95..3b52e49a 100644 --- a/webscout/Provider/toolbaz.py +++ b/webscout/Provider/toolbaz.py @@ -1,20 +1,25 @@ -import re -from curl_cffi.requests import Session -from curl_cffi import CurlError -import uuid import base64 import json import random +import re import string import time +import uuid from datetime import datetime -from typing import Any, Dict, Optional, Generator, Union, List +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session from webscout import exceptions -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) + class Toolbaz(Provider): """ @@ -37,7 +42,7 @@ class Toolbaz(Provider): "grok-4-fast", "grok-4.1-fast", - + "toolbaz-v4.5-fast", "toolbaz_v4", "toolbaz_v3.5_pro", @@ -48,7 +53,7 @@ class Toolbaz(Provider): "Llama-4-Maverick", "Llama-3.3-70B", - + "mixtral_8x22b", "L3-70B-Euryale-v2.1", "midnight-rose", @@ -60,12 +65,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 600, # Note: max_tokens is not directly used by the API timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "gemini-2.0-flash", system_prompt: str = "You are a helpful AI assistant." # Note: system_prompt is not directly used by the API ): @@ -156,7 +161,7 @@ def get_auth(self): } # Use curl_cffi session post WITHOUT impersonate for token request resp = self.session.post( - "https://data.toolbaz.com/token.php", + "https://data.toolbaz.com/token.php", data=data # Removed impersonate="chrome110" for this specific request ) @@ -181,10 +186,11 @@ def ask( self, prompt: str, stream: bool = False, - raw: bool = False, # Kept for compatibility, but output is always dict/string + raw: bool = False, optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: """Sends a prompt to the Toolbaz API and returns the response.""" if optimizer and optimizer not in self.__available_optimizers: raise exceptions.FailedToGenerateResponseError(f"Optimizer is not one of {self.__available_optimizers}") @@ -196,7 +202,7 @@ def ask( ) # get_auth now raises exceptions on failure - auth = self.get_auth() + auth = self.get_auth() # No need to check if auth is None, as an exception would have been raised data = { @@ -264,7 +270,7 @@ def for_non_stream(): resp.raise_for_status() # Use response.text which is already decoded - text = resp.text + text = resp.text # Remove [model: ...] tags text = re.sub(r"\[model:.*?\]", "", text) @@ -287,6 +293,7 @@ def chat( optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, # Added raw parameter + **kwargs: Any, ) -> Union[str, Generator[str, None, None]]: """Generates a response from the Toolbaz API.""" def for_stream_chat(): @@ -319,7 +326,7 @@ def for_non_stream_chat(): return for_stream_chat() if stream else for_non_stream_chat() - def get_message(self, response: Dict[str, Any]) -> str: + def get_message(self, response: Response) -> str: """Extract the message from the response. Args: @@ -328,13 +335,14 @@ def get_message(self, response: Dict[str, Any]) -> str: Returns: str: Message extracted """ - assert isinstance(response, dict), "Response should be of dict data-type only" + if not isinstance(response, dict): + return str(response) return response.get("text", "") # Example usage if __name__ == "__main__": # Ensure curl_cffi is installed - from rich import print # Use rich print if available + from rich import print # Use rich print if available print("-" * 80) print(f"{'Model':<50} {'Status':<10} {'Response'}") print("-" * 80) @@ -346,10 +354,11 @@ def get_message(self, response: Dict[str, Any]) -> str: response_stream = test_ai.chat("Say 'Hello' in one word", stream=True) response_text = "" # print(f"\r{model:<50} {'Streaming...':<10}", end="", flush=True) - for chunk in response_stream: - response_text += chunk - # Optional: print chunks for visual feedback - # print(chunk, end="", flush=True) + if hasattr(response_stream, "__iter__") and not isinstance(response_stream, (str, bytes)): + for chunk in response_stream: + response_text += chunk + else: + response_text = str(response_stream) if response_text and len(response_text.strip()) > 0: status = "✓" @@ -369,4 +378,4 @@ def get_message(self, response: Dict[str, Any]) -> str: except Exception as e: # Print full error for debugging - print(f"\r{model:<50} {'✗':<10} Error: {str(e)}") \ No newline at end of file + print(f"\r{model:<50} {'✗':<10} Error: {str(e)}") diff --git a/webscout/Provider/turboseek.py b/webscout/Provider/turboseek.py index 0fe1cb5f..cf71f6b2 100644 --- a/webscout/Provider/turboseek.py +++ b/webscout/Provider/turboseek.py @@ -1,16 +1,21 @@ import re -from typing import Optional, Union, Any, AsyncGenerator, Dict -from curl_cffi.requests import Session +from typing import Any, Generator, Optional, Union + from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) from webscout.litagent import LitAgent + class TurboSeek(Provider): """ This class provides methods for interacting with the TurboSeek API. @@ -23,12 +28,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 600, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "Llama 3.1 70B" # Note: model parameter is not used by the API endpoint ): """Instantiates TurboSeek @@ -96,34 +101,34 @@ def _html_to_markdown(text: str) -> str: """Convert basic HTML tags to Markdown.""" if not text: return "" - + # Unescape HTML entities first import html text = html.unescape(text) # Headers text = re.sub(r']*>(.*?)', r'\n# \1\n', text) - + # Lists text = re.sub(r']*>(.*?)', r'\n* \1', text) text = re.sub(r'<(ul|ol)[^>]*>', r'\n', text) text = re.sub(r'', r'\n', text) - + # Paragraphs and Breaks text = re.sub(r'

', r'\n\n', text) text = re.sub(r']*>', r'\n', text) text = re.sub(r'', r'\n', text) - + # Bold and Italic text = re.sub(r'<(strong|b)[^>]*>(.*?)', r'**\2**', text) text = re.sub(r'<(em|i)[^>]*>(.*?)', r'*\2*', text) - + # Remove structural tags text = re.sub(r']*>', '', text, flags=re.IGNORECASE) - + # Final cleanup of remaining tags text = re.sub(r'<[^>]*>', '', text) - + return text @staticmethod @@ -139,9 +144,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> dict: + **kwargs: Any, + ) -> Response: """Chat with AI """ conversation_prompt = self.conversation.gen_complete_prompt(prompt) @@ -163,9 +169,9 @@ def ask( def for_stream(): try: response = self.session.post( - self.chat_endpoint, - json=payload, - stream=True, + self.chat_endpoint, + json=payload, + stream=True, timeout=self.timeout, impersonate="chrome120" ) @@ -173,23 +179,23 @@ def for_stream(): raise exceptions.FailedToGenerateResponseError( f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}" ) - + streaming_text = "" # The API returns raw HTML chunks now, no "data:" prefix processed_stream = sanitize_stream( data=response.iter_content(chunk_size=None), - intro_value=None, + intro_value=None, to_json=False, strip_chars='', # Disable default lstrip to preserve spacing content_extractor=self._turboseek_extractor, yield_raw_on_error=True, raw=raw ) - + for content_chunk in processed_stream: if content_chunk is None: continue - + if raw: yield content_chunk else: @@ -202,7 +208,7 @@ def for_stream(): streaming_text += clean_chunk self.last_response.update(dict(text=streaming_text)) yield dict(text=clean_chunk) - + if not raw and streaming_text: self.conversation.update_chat_history( prompt, streaming_text @@ -219,15 +225,15 @@ def for_non_stream(): # We use ask(..., raw=True) internally or just the local for_stream # Actually, let's just make a sub-call response = self.session.post( - self.chat_endpoint, - json=payload, + self.chat_endpoint, + json=payload, timeout=self.timeout, impersonate="chrome120" ) full_html = response.text except Exception as e: raise exceptions.FailedToGenerateResponseError(f"Failed to get non-stream response: {e}") from e - + # Convert full HTML to Markdown final_text = self._html_to_markdown(full_html).strip() self.last_response = {"text": final_text} @@ -239,10 +245,11 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, # Added raw parameter - ) -> str: + **kwargs: Any, + ) -> Union[str, Generator[str, None, None]]: """Generate response `str` Args: prompt (str): Prompt to be send. @@ -275,23 +282,24 @@ def for_non_stream(): return self.get_message(result) return for_stream() if stream else for_non_stream() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """Retrieves message only from response Args: - response (dict): Response generated by `self.ask` + response (Response): Response generated by `self.ask` Returns: str: Message extracted """ - assert isinstance(response, dict), "Response should be of dict data-type only" + if not isinstance(response, dict): + return str(response) # Unicode escapes are handled by json.loads within sanitize_stream - return response.get("text", "") + return response.get("text", "") if __name__ == '__main__': import sys ai = TurboSeek(timeout=60) - + # helper for safe printing on windows def safe_print(text, end="\n"): try: @@ -302,9 +310,16 @@ def safe_print(text, end="\n"): safe_print("\n=== Testing Non-Streaming ===") response = ai.chat("How can I get a 6 pack in 3 months?", stream=False) - safe_print(response) - + if isinstance(response, str): + safe_print(response) + else: + safe_print(str(response)) + safe_print("\n=== Testing Streaming ===") - for chunk in ai.chat("How can I get a 6 pack in 3 months?", stream=True): - safe_print(chunk, end="") - safe_print("") \ No newline at end of file + stream_resp = ai.chat("How can I get a 6 pack in 3 months?", stream=True) + if hasattr(stream_resp, "__iter__") and not isinstance(stream_resp, (str, bytes)): + for chunk in stream_resp: + safe_print(chunk, end="") + else: + safe_print(str(stream_resp)) + safe_print("") diff --git a/webscout/Provider/typefully.py b/webscout/Provider/typefully.py index 2265beb3..467e1856 100644 --- a/webscout/Provider/typefully.py +++ b/webscout/Provider/typefully.py @@ -1,13 +1,12 @@ -from typing import Any, Dict +from typing import Any, Dict, Generator, Optional, Union + +from curl_cffi import CurlError +from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Conversation, Optimizers, sanitize_stream from webscout.litagent import LitAgent -from curl_cffi.requests import Session -from curl_cffi import CurlError class TypefullyAI(Provider): @@ -24,12 +23,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 600, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, system_prompt: str = "You're a helpful assistant.", model: str = "openai:gpt-4o-mini", ): @@ -77,7 +76,6 @@ def __init__( @staticmethod def _typefully_extractor(chunk) -> str: """Extracts content from Typefully AI SSE format.""" - import re import json # Handle parsed JSON objects (when to_json=True) @@ -120,9 +118,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Dict[str, Any]: + **kwargs: Any, + ) -> Response: conversation_prompt = self.conversation.gen_complete_prompt(prompt) if optimizer: if optimizer in self.__available_optimizers: @@ -191,10 +190,11 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, - ) -> str: + **kwargs: Any, + ) -> Union[str, Generator[str, None, None]]: def for_stream(): for response in self.ask( prompt, True, raw=raw, optimizer=optimizer, conversationally=conversationally @@ -219,8 +219,9 @@ def for_non_stream(): return for_stream() if stream else for_non_stream() - def get_message(self, response: dict) -> str: - assert isinstance(response, dict), "Response should be of dict data-type only" + def get_message(self, response: Response) -> str: + if not isinstance(response, dict): + return str(response) text = response.get("text", "") try: formatted_text = text.replace("\\n", "\n").replace("\\n\\n", "\n\n") @@ -240,8 +241,12 @@ def get_message(self, response: dict) -> str: test_ai = TypefullyAI(model=model, timeout=60) response_stream = test_ai.chat("Say 'Hello' in one word", stream=True) response_text = "" - for chunk in response_stream: - response_text += chunk + if hasattr(response_stream, "__iter__") and not isinstance(response_stream, (str, bytes)): + for chunk in response_stream: + response_text += chunk + else: + response_text = str(response_stream) + if response_text and len(response_text.strip()) > 0: status = "OK" clean_text = response_text.strip() diff --git a/webscout/Provider/x0gpt.py b/webscout/Provider/x0gpt.py index 47982499..4bcf252e 100644 --- a/webscout/Provider/x0gpt.py +++ b/webscout/Provider/x0gpt.py @@ -1,16 +1,22 @@ -from typing import Generator, Union, Any, Dict +from typing import Any, Dict, Generator, Optional, Union from uuid import uuid4 + from curl_cffi import CurlError + +# Import HTTPVersion enum +from curl_cffi.const import CurlHttpVersion from curl_cffi.requests import Session -from webscout.AIutel import Optimizers -from webscout.AIutel import Conversation -from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions +from webscout.AIbase import Provider, Response +from webscout.AIutel import ( # Import sanitize_stream + AwesomePrompts, + Conversation, + Optimizers, + sanitize_stream, +) from webscout.litagent import LitAgent -# Import HTTPVersion enum -from curl_cffi.const import CurlHttpVersion + class X0GPT(Provider): """ @@ -34,12 +40,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 600, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, system_prompt: str = "You are a helpful assistant.", model: str = "UNKNOWN" ): @@ -64,7 +70,7 @@ def __init__( 'You are a friendly assistant.' """ # Initialize curl_cffi Session instead of requests.Session - self.session = Session() + self.session = Session() self.is_conversation = is_conversation self.max_tokens_to_sample = max_tokens self.api_endpoint = "https://x0-gpt.devwtf.in/api/stream/reply" @@ -121,9 +127,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: """ Sends a prompt to the x0-gpt.devwtf.in API and returns the response. @@ -179,7 +186,7 @@ def for_stream(): raise exceptions.FailedToGenerateResponseError( f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}" ) - + streaming_response = "" # Use sanitize_stream with regex-based extraction and filtering processed_stream = sanitize_stream( @@ -204,7 +211,7 @@ def for_stream(): # Always yield as string, even in raw mode if isinstance(content_chunk, bytes): content_chunk = content_chunk.decode('utf-8', errors='ignore') - + if raw: yield content_chunk else: @@ -246,9 +253,10 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, # Added raw parameter + **kwargs: Any, ) -> Union[str, Generator[str, None, None]]: """ Generates a response from the X0GPT API. @@ -321,5 +329,8 @@ def get_message(self, response: dict) -> str: from rich import print ai = X0GPT(timeout=5000) response = ai.chat("write a poem about AI", stream=True, raw=False) - for chunk in response: - print(chunk, end="", flush=True) \ No newline at end of file + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end="", flush=True) + else: + print(response) diff --git a/webscout/Provider/yep.py b/webscout/Provider/yep.py index ebf8148d..d7e5fc1a 100644 --- a/webscout/Provider/yep.py +++ b/webscout/Provider/yep.py @@ -1,14 +1,14 @@ import uuid +from typing import Any, Dict, Generator, Optional, TypeVar, Union + from curl_cffi import CurlError from curl_cffi.requests import Session -from typing import Any, Dict, Optional, Generator, Union, List, TypeVar -from webscout.AIutel import Optimizers -from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream -from webscout.AIbase import Provider from webscout import exceptions -from webscout.litagent import LitAgent +from webscout.AIbase import Provider, Response +from webscout.AIutel import AwesomePrompts, Optimizers, sanitize_stream # Import sanitize_stream from webscout.conversation import Conversation +from webscout.litagent import LitAgent T = TypeVar('T') @@ -29,12 +29,12 @@ def __init__( is_conversation: bool = True, max_tokens: int = 1280, timeout: int = 30, - intro: str = None, - filepath: str = None, + intro: Optional[str] = None, + filepath: Optional[str] = None, update_file: bool = True, proxies: dict = {}, history_offset: int = 10250, - act: str = None, + act: Optional[str] = None, model: str = "DeepSeek-R1-Distill-Qwen-32B", temperature: float = 0.6, top_p: float = 0.7, @@ -57,7 +57,7 @@ def __init__( ) # Initialize curl_cffi Session instead of cloudscraper - self.session = Session() + self.session = Session() self.is_conversation = is_conversation self.max_tokens_to_sample = max_tokens self.chat_endpoint = "https://api.yep.com/v1/chat/completions" @@ -87,7 +87,7 @@ def __init__( "Sec-CH-UA-Platform": f'"{self.fingerprint["platform"]}"', "User-Agent": self.fingerprint["user_agent"], } - + # Create session cookies with unique identifiers self.cookies = {"__Host-session": uuid.uuid4().hex, '__cf_bm': uuid.uuid4().hex} @@ -114,13 +114,13 @@ def __init__( def refresh_identity(self, browser: str = None): """ Refreshes the browser identity fingerprint. - + Args: browser: Specific browser to use for the new fingerprint """ browser = browser or self.fingerprint.get("browser_type", "chrome") self.fingerprint = self.agent.generate_fingerprint(browser) - + # Update headers with new fingerprint self.headers.update({ "Accept": self.fingerprint["accept"], @@ -129,13 +129,13 @@ def refresh_identity(self, browser: str = None): "Sec-CH-UA-Platform": f'"{self.fingerprint["platform"]}"', "User-Agent": self.fingerprint["user_agent"], }) - + # Update session headers self.session.headers.update(self.headers) - + # Generate new cookies (will be passed in requests) self.cookies = {"__Host-session": uuid.uuid4().hex, '__cf_bm': uuid.uuid4().hex} - + return self.fingerprint def ask( @@ -143,9 +143,10 @@ def ask( prompt: str, stream: bool = False, raw: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, - ) -> Union[Dict[str, Any], Generator]: + **kwargs: Any, + ) -> Response: """ Sends a prompt to the Yep API and returns the response. @@ -237,7 +238,7 @@ def for_non_stream(): ) if raw: return response.text - + # Use sanitize_stream to parse the non-streaming JSON response processed_stream = sanitize_stream( data=response.text, @@ -252,7 +253,7 @@ def for_non_stream(): if raw: return content content = content if isinstance(content, str) else "" - + if content: self.conversation.update_chat_history(prompt, content) return {"text": content} @@ -269,9 +270,10 @@ def chat( self, prompt: str, stream: bool = False, - optimizer: str = None, + optimizer: Optional[str] = None, conversationally: bool = False, raw: bool = False, # Added raw parameter + **kwargs: Any, ) -> Union[str, Generator[str, None, None]]: """ Initiates a chat with the Yep API using the provided prompt. @@ -308,7 +310,7 @@ def for_non_stream(): return for_stream() if stream else for_non_stream() - def get_message(self, response: dict) -> str: + def get_message(self, response: Response) -> str: """ Extracts the message content from the API response. @@ -319,36 +321,19 @@ def get_message(self, response: dict) -> str: Extracts and returns the message content from the response. """ if isinstance(response, dict): - return response["text"] + return response.get("text", "") elif isinstance(response, (str, bytes)): - return response + return str(response) else: - raise TypeError(f"Unexpected response type: {type(response)}") + return str(response) if __name__ == "__main__": - # print("-" * 80) - # print(f"{'Model':<50} {'Status':<10} {'Response'}") - # print("-" * 80) - - # for model in YEPCHAT.AVAILABLE_MODELS: - # try: - # test_ai = YEPCHAT(model=model, timeout=60) - # response = test_ai.chat("Say 'Hello' in one word") - # response_text = response - - # if response_text and len(response_text.strip()) > 0: - # status = "✓" - # # Truncate response if too long - # display_text = response_text.strip()[:50] + "..." if len(response_text.strip()) > 50 else response_text.strip() - # else: - # status = "✗" - # display_text = "Empty or invalid response" - # print(f"{model:<50} {status:<10} {display_text}") - # except Exception as e: - # print(f"{model:<50} {'✗':<10} {str(e)}") ai = YEPCHAT(model="DeepSeek-R1-Distill-Qwen-32B", timeout=60) response = ai.chat("Say 'Hello' in one word", raw=False, stream=True) - for chunk in response: + if hasattr(response, "__iter__") and not isinstance(response, (str, bytes)): + for chunk in response: + print(chunk, end='', flush=True) + else: + print(response) - print(chunk, end='', flush=True) \ No newline at end of file diff --git a/webscout/__init__.py b/webscout/__init__.py index 67cb7b54..d61e574a 100644 --- a/webscout/__init__.py +++ b/webscout/__init__.py @@ -1,30 +1,31 @@ # webscout/__init__.py -from .search import * -from .version import __version__ +from .AIauto import * # noqa: F403 +from .AIutel import * # noqa: F403 +from .client import Client +from .Extra import * # noqa: F403 +from .litagent import LitAgent +from .models import model +from .optimizers import * from .Provider import * -from .AIauto import * +from .Provider.AISEARCH import * +from .Provider.STT import * # noqa: F403 from .Provider.TTI import * from .Provider.TTS import * -from .Provider.AISEARCH import * -from .Provider.STT import * -from .Extra import * -from .optimizers import * -from .swiftcli import * -from .litagent import LitAgent -from .client import Client from .scout import * +from .search import * +from .swiftcli import * +from .update_checker import check_for_updates +from .version import __version__ from .zeroart import * -from .AIutel import * useragent = LitAgent() -# Add update checker -from .update_checker import check_for_updates + + try: update_message = check_for_updates() if update_message: print(update_message) -except Exception as e: +except Exception: pass -# Import models for easy access -from .models import model \ No newline at end of file + diff --git a/webscout/cli.py b/webscout/cli.py index e53a4331..e4e707ca 100644 --- a/webscout/cli.py +++ b/webscout/cli.py @@ -1,25 +1,29 @@ -from .swiftcli import CLI, option, table_output, panel_output +import sys +from typing import Any, Dict, List, Optional, Type + +from rich import print as rprint +from rich.console import Console +from rich.panel import Panel +from rich.table import Table + from .search import ( - DuckDuckGoSearch, - YepSearch, - BingSearch, - YahooSearch, + BaseSearch, + BingSearch, Brave, + DuckDuckGoSearch, Mojeek, + Wikipedia, + YahooSearch, Yandex, - Wikipedia + YepSearch, ) +from .swiftcli import CLI, option from .version import __version__ -from rich.console import Console -from rich.table import Table -from rich.panel import Panel -from rich import print as rprint -import sys console = Console() # Engine mapping -ENGINES = { +ENGINES: Dict[str, Type[BaseSearch]] = { "ddg": DuckDuckGoSearch, "duckduckgo": DuckDuckGoSearch, "bing": BingSearch, @@ -28,32 +32,32 @@ "mojeek": Mojeek, "yandex": Yandex, "wikipedia": Wikipedia, - "yep": YepSearch + "yep": YepSearch, } -def _get_engine(name): +def _get_engine(name: str) -> BaseSearch: cls = ENGINES.get(name.lower()) if not cls: rprint(f"[bold red]Error: Engine '{name}' not supported.[/bold red]") rprint(f"Available engines: {', '.join(sorted(set(e for e in ENGINES.keys())))}") sys.exit(1) - return cls() + return cls() # type: ignore[arg-type] -def _print_data(data, title="Search Results"): +def _print_data(data: Any, title: str = "Search Results") -> None: """Prints data in a beautiful table.""" if not data: rprint("[bold yellow]No results found.[/bold yellow]") return table = Table(title=title, show_header=True, header_style="bold magenta", show_lines=True) - + if isinstance(data, list) and len(data) > 0: if isinstance(data[0], dict): keys = list(data[0].keys()) table.add_column("#", style="dim", width=4) for key in keys: table.add_column(key.capitalize()) - + for i, item in enumerate(data, 1): row = [str(i)] for key in keys: @@ -73,44 +77,51 @@ def _print_data(data, title="Search Results"): console.print(table) -def _print_weather(data): +def _print_weather(data: Dict[str, Any]) -> None: """Prints weather data in a clean panel.""" - current = data.get("current") - if not current: - rprint(f"[bold blue]Weather data:[/bold blue] {data}") - return - + # Be defensive when reading weather payloads + current = data.get("current") or {} + location = data.get("location", "Unknown") + + temp = current.get('temperature_c', 'N/A') + feels_like = current.get('feels_like_c', 'N/A') + condition = current.get('condition', 'N/A') + humidity = current.get('humidity', 'N/A') + wind_speed = current.get('wind_speed_ms', 'N/A') + wind_dir = current.get('wind_direction', 'N/A') + weather_info = ( - f"[bold blue]Location:[/bold blue] {data['location']}\n" - f"[bold blue]Temperature:[/bold blue] {current['temperature_c']}°C (Feels like {current['feels_like_c']}°C)\n" - f"[bold blue]Condition:[/bold blue] {current['condition']}\n" - f"[bold blue]Humidity:[/bold blue] {current['humidity']}%\n" - f"[bold blue]Wind:[/bold blue] {current['wind_speed_ms']} m/s {current['wind_direction']}°" + f"[bold blue]Location:[/bold blue] {location}\n" + f"[bold blue]Temperature:[/bold blue] {temp}°C (Feels like {feels_like}°C)\n" + f"[bold blue]Condition:[/bold blue] {condition}\n" + f"[bold blue]Humidity:[/bold blue] {humidity}%\n" + f"[bold blue]Wind:[/bold blue] {wind_speed} m/s {wind_dir}°" ) - + panel = Panel(weather_info, title="Current Weather", border_style="green") console.print(panel) - - if "daily_forecast" in data: + + if isinstance(data.get("daily_forecast"), list): forecast_table = Table(title="5-Day Forecast", show_header=True, header_style="bold cyan") forecast_table.add_column("Date") forecast_table.add_column("Condition") forecast_table.add_column("High") forecast_table.add_column("Low") - - for day in data["daily_forecast"][:5]: - forecast_table.add_row( - day['date'], - day['condition'], - f"{day['max_temp_c']:.1f}°C", - f"{day['min_temp_c']:.1f}°C" - ) + + for day in data.get("daily_forecast", [])[:5]: + date = day.get('date', 'N/A') + condition = day.get('condition', 'N/A') + max_temp = day.get('max_temp_c') + min_temp = day.get('min_temp_c') + max_temp_str = f"{max_temp:.1f}°C" if isinstance(max_temp, (int, float)) else str(max_temp) + min_temp_str = f"{min_temp:.1f}°C" if isinstance(min_temp, (int, float)) else str(min_temp) + forecast_table.add_row(date, condition, max_temp_str, min_temp_str) console.print(forecast_table) -app = CLI(name="webscout", help="Search the web with a simple UI", version=__version__) +app: CLI = CLI(name="webscout", help="Search the web with a simple UI", version=__version__) @app.command() -def version(): +def version() -> None: """Show the version of webscout.""" rprint(f"[bold cyan]webscout version:[/bold cyan] {__version__}") @@ -121,14 +132,21 @@ def version(): @option("--safesearch", "-s", help="SafeSearch setting", default="moderate") @option("--timelimit", "-t", help="Time limit for results", default=None) @option("--max-results", "-m", help="Maximum number of results", type=int, default=10) -def text(keywords: str, engine: str, region: str, safesearch: str, timelimit: str, max_results: int): +def text( + keywords: str, + engine: str, + region: Optional[str] = None, + safesearch: str = "moderate", + timelimit: Optional[str] = None, + max_results: int = 10, +) -> None: """Perform a text search.""" try: search_engine = _get_engine(engine) # Handle region defaults if not provided if region is None: region = "wt-wt" if engine.lower() in ["ddg", "duckduckgo"] else "us" - + # Most engines use .text(), some use .search() or .run() if hasattr(search_engine, 'text'): results = search_engine.text(keywords, region=region, safesearch=safesearch, max_results=max_results) @@ -136,7 +154,7 @@ def text(keywords: str, engine: str, region: str, safesearch: str, timelimit: st results = search_engine.run(keywords, region=region, safesearch=safesearch, max_results=max_results) else: results = search_engine.search(keywords, max_results=max_results) - + _print_data(results, title=f"{engine.upper()} Text Search: {keywords}") except Exception as e: rprint(f"[bold red]Error:[/bold red] {str(e)}") @@ -145,7 +163,7 @@ def text(keywords: str, engine: str, region: str, safesearch: str, timelimit: st @option("--keywords", "-k", help="Search keywords", required=True) @option("--engine", "-e", help="Search engine (ddg, bing, yahoo)", default="ddg") @option("--max-results", "-m", help="Maximum number of results", type=int, default=10) -def images(keywords: str, engine: str, max_results: int): +def images(keywords: str, engine: str, max_results: int) -> None: """Perform an images search.""" try: search_engine = _get_engine(engine) @@ -158,7 +176,7 @@ def images(keywords: str, engine: str, max_results: int): @option("--keywords", "-k", help="Search keywords", required=True) @option("--engine", "-e", help="Search engine (ddg, yahoo)", default="ddg") @option("--max-results", "-m", help="Maximum number of results", type=int, default=10) -def videos(keywords: str, engine: str, max_results: int): +def videos(keywords: str, engine: str, max_results: int) -> None: """Perform a videos search.""" try: search_engine = _get_engine(engine) @@ -171,7 +189,7 @@ def videos(keywords: str, engine: str, max_results: int): @option("--keywords", "-k", help="Search keywords", required=True) @option("--engine", "-e", help="Search engine (ddg, bing, yahoo)", default="ddg") @option("--max-results", "-m", help="Maximum number of results", type=int, default=10) -def news(keywords: str, engine: str, max_results: int): +def news(keywords: str, engine: str, max_results: int) -> None: """Perform a news search.""" try: search_engine = _get_engine(engine) @@ -183,7 +201,7 @@ def news(keywords: str, engine: str, max_results: int): @app.command() @option("--location", "-l", help="Location to get weather for", required=True) @option("--engine", "-e", help="Search engine (ddg, yahoo)", default="ddg") -def weather(location: str, engine: str): +def weather(location: str, engine: str) -> None: """Get weather information.""" try: search_engine = _get_engine(engine) @@ -195,7 +213,7 @@ def weather(location: str, engine: str): @app.command() @option("--keywords", "-k", help="Search keywords", required=True) @option("--engine", "-e", help="Search engine (ddg, yahoo)", default="ddg") -def answers(keywords: str, engine: str): +def answers(keywords: str, engine: str) -> None: """Perform an answers search.""" try: search_engine = _get_engine(engine) @@ -207,21 +225,16 @@ def answers(keywords: str, engine: str): @app.command() @option("--query", "-q", help="Search query", required=True) @option("--engine", "-e", help="Search engine (ddg, bing, yahoo, yep)", default="ddg") -def suggestions(query: str, engine: str): +def suggestions(query: str, engine: str) -> None: """Get search suggestions.""" try: search_engine = _get_engine(engine) - # Some engines use 'keywords', some 'query' - if engine.lower() in ["bing", "yep"]: - results = search_engine.suggestions(query) - else: - results = search_engine.suggestions(query) - - # Format suggestions + results = search_engine.suggestions(query) + + # Format suggestions (Bing-style dicts) if isinstance(results, list) and results and isinstance(results[0], dict): - # Bing format results = [r.get("suggestion", str(r)) for r in results] - + _print_data(results, title=f"{engine.upper()} Suggestions: {query}") except Exception as e: rprint(f"[bold red]Error:[/bold red] {str(e)}") @@ -231,7 +244,7 @@ def suggestions(query: str, engine: str): @option("--from", "-f", help="Language to translate from", default=None) @option("--to", "-t", help="Language to translate to", default="en") @option("--engine", "-e", help="Search engine (ddg, yahoo)", default="ddg") -def translate(keywords: str, from_: str, to: str, engine: str): +def translate(keywords: str, from_: Optional[str] = None, to: str = "en", engine: str = "ddg") -> None: """Perform translation.""" try: search_engine = _get_engine(engine) @@ -245,7 +258,7 @@ def translate(keywords: str, from_: str, to: str, engine: str): @option("--place", "-p", help="Place name") @option("--radius", "-r", help="Search radius (km)", type=int, default=0) @option("--engine", "-e", help="Search engine (ddg, yahoo)", default="ddg") -def maps(keywords: str, place: str, radius: int, engine: str): +def maps(keywords: str, place: Optional[str] = None, radius: int = 0, engine: str = "ddg") -> None: """Perform a maps search.""" try: search_engine = _get_engine(engine) @@ -259,9 +272,10 @@ def maps(keywords: str, place: str, radius: int, engine: str): @option("--keywords", "-k", help="Search keywords", required=True) @option("--engine", "-e", help="Search engine", default="ddg") @option("--max-results", "-m", help="Maximum results", type=int, default=10) -def search(keywords: str, engine: str, max_results: int): +def search(keywords: str, engine: str, max_results: int) -> None: """Unified search command across all engines.""" - text.run(keywords=keywords, engine=engine, max_results=max_results) + # Call the local `text` function implementation (not a command object) + text(keywords=keywords, engine=engine, max_results=max_results) def main(): """Main entry point for the CLI.""" @@ -272,4 +286,4 @@ def main(): sys.exit(1) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/webscout/client.py b/webscout/client.py index 2ffc7b8e..bdbbb7d5 100644 --- a/webscout/client.py +++ b/webscout/client.py @@ -12,29 +12,20 @@ - Full streaming support """ -import time -import uuid -import random -import inspect +import difflib import importlib +import inspect import pkgutil -import difflib -from typing import List, Dict, Optional, Union, Generator, Any, Type, Tuple, Set +import random +from typing import Any, Dict, Generator, List, Optional, Set, Tuple, Type, Union -from webscout.Provider.OPENAI import * -from webscout.Provider.OPENAI.base import OpenAICompatibleProvider, BaseCompletions, BaseChat +from webscout.Provider.OPENAI.base import BaseChat, BaseCompletions, OpenAICompatibleProvider from webscout.Provider.OPENAI.utils import ( ChatCompletion, ChatCompletionChunk, - Choice, - ChoiceDelta, - ChatCompletionMessage, - CompletionUsage, ) - -from webscout.Provider.TTI import * -from webscout.Provider.TTI.base import TTICompatibleProvider, BaseImages -from webscout.Provider.TTI.utils import ImageData, ImageResponse +from webscout.Provider.TTI.base import BaseImages, TTICompatibleProvider +from webscout.Provider.TTI.utils import ImageResponse def load_openai_providers() -> Tuple[Dict[str, Type[OpenAICompatibleProvider]], Set[str]]: @@ -359,7 +350,7 @@ def create( first_chunk = next(response) self._last_provider = resolved_provider.__name__ - def chained_gen(first, rest, pname): + def chained_gen(first, rest, pname) -> Any: if self._client.print_provider_info: print(f"\033[1;34m{pname}:{resolved_model}\033[0m\n") yield first @@ -718,7 +709,7 @@ def generate( return response except Exception: continue - raise RuntimeError(f"All image providers failed.") + raise RuntimeError("All image providers failed.") def create(self, **kwargs) -> ImageResponse: """Alias for generate.""" diff --git a/webscout/conversation.py b/webscout/conversation.py index 11c48b43..40e38a87 100644 --- a/webscout/conversation.py +++ b/webscout/conversation.py @@ -25,7 +25,7 @@ def __init__( self, status: bool = True, max_tokens: int = 600, - filepath: str = None, + filepath: Optional[str] = None, update_file: bool = True, ): """Initializes Conversation @@ -88,7 +88,7 @@ def __trim_chat_history(self, chat_history: str, intro: str) -> str: else: return chat_history - def gen_complete_prompt(self, prompt: str, intro: str = None) -> str: + def gen_complete_prompt(self, prompt: str, intro: Optional[str] = None) -> str: """Generates a kinda like incomplete conversation Args: diff --git a/webscout/exceptions.py b/webscout/exceptions.py index 5b823ae4..42b017f4 100644 --- a/webscout/exceptions.py +++ b/webscout/exceptions.py @@ -90,6 +90,18 @@ class FailedToGenerateResponseError(WebscoutE): """ pass +class ProviderConnectionError(WebscoutE): + """ + Exception raised when there are issues connecting to a specific provider. + """ + pass + +class InvalidOptimizerError(WebscoutE): + """ + Exception raised when an invalid or unavailable optimizer is requested. + """ + pass + class InvalidAuthenticationError(Exception): """Custom exception for authentication errors (e.g., invalid API key, cookies).""" pass @@ -359,4 +371,4 @@ def __init__(self, video_id, requested_language_codes, transcript_data): super().__init__(video_id, message.format( requested_language_codes=requested_language_codes, transcript_data=str(transcript_data) - )) \ No newline at end of file + )) diff --git a/webscout/litagent/__init__.py b/webscout/litagent/__init__.py index 43987e2d..ef16e095 100644 --- a/webscout/litagent/__init__.py +++ b/webscout/litagent/__init__.py @@ -4,7 +4,7 @@ Examples: >>> from webscout import LitAgent >>> agent = LitAgent() ->>> +>>> >>> # Get random user agents >>> agent.random() # Random agent from any browser >>> agent.mobile() # Random mobile device agent @@ -22,8 +22,8 @@ """ from .agent import LitAgent -from .constants import BROWSERS, OS_VERSIONS, DEVICES, FINGERPRINTS +from .constants import BROWSERS, DEVICES, FINGERPRINTS, OS_VERSIONS agent = LitAgent() -__all__ = ['LitAgent', 'agent', 'BROWSERS', 'OS_VERSIONS', 'DEVICES', 'FINGERPRINTS'] \ No newline at end of file +__all__ = ['LitAgent', 'agent', 'BROWSERS', 'OS_VERSIONS', 'DEVICES', 'FINGERPRINTS'] diff --git a/webscout/litagent/agent.py b/webscout/litagent/agent.py index 8dca6245..d75e024d 100644 --- a/webscout/litagent/agent.py +++ b/webscout/litagent/agent.py @@ -2,7 +2,7 @@ LitAgent: Advanced User Agent Generation and Management System. This module provides a robust and flexible system for generating realistic, -modern user agents and managing browser fingerprints for web scraping and +modern user agents and managing browser fingerprints for web scraping and automation purposes. """ @@ -10,7 +10,7 @@ import random import threading from datetime import datetime -from typing import Any, Dict, List, Optional, Union, Tuple +from typing import Any, Dict, List, Optional from webscout.litagent.constants import BROWSERS, DEVICES, FINGERPRINTS, OS_VERSIONS @@ -18,7 +18,7 @@ class LitAgent: """ A powerful and modern user agent generator for web scraping and automation. - + LitAgent provides tools for generating randomized but realistic user agent strings, managing proxy pools, rotating IPs, and simulating browser fingerprints to avoid detection and rate limiting. @@ -43,7 +43,7 @@ def __init__(self, thread_safe: bool = False): """ self.thread_safe = thread_safe self.lock = threading.RLock() if thread_safe else None - + # Internal state self._blacklist: set = set() self._whitelist: set = set() @@ -54,7 +54,7 @@ def __init__(self, thread_safe: bool = False): self._proxy_index: int = 0 self._history: List[str] = [] self._refresh_timer: Optional[threading.Timer] = None - + # Usage statistics self._stats = { "total_generated": 100, @@ -92,7 +92,7 @@ def _generate_agents(self, count: int) -> List[str]: platform = f"X11; Linux {os_ver}" agent = f"Mozilla/5.0 ({platform}) AppleWebKit/537.36 (KHTML, like Gecko) " - + if browser == 'chrome': agent += f"Chrome/{version}.0.0.0 Safari/537.36" elif browser == 'firefox': @@ -105,7 +105,7 @@ def _generate_agents(self, count: int) -> List[str]: agent += f"Chrome/{version}.0.0.0 Safari/537.36 Brave/{version}.0.0.0" elif browser == 'vivaldi': agent += f"Chrome/{version}.0.0.0 Safari/537.36 Vivaldi/{version}.0.{random.randint(1000, 9999)}" - + elif browser == 'safari': device = random.choice(['mac', 'ios']) if device == 'mac': @@ -115,7 +115,7 @@ def _generate_agents(self, count: int) -> List[str]: ver = random.choice(OS_VERSIONS.get('ios', ["17_0"])) device_name = random.choice(['iPhone', 'iPad']) agent = f"Mozilla/5.0 ({device_name}; CPU OS {ver} like Mac OS X) " - + agent += f"AppleWebKit/{version}.1.15 (KHTML, like Gecko) Version/{version//10}.0 Safari/{version}.1.15" agents.append(agent) @@ -163,7 +163,7 @@ def random(self) -> str: if not pool: # Fallback if somehow empty pool = self._generate_agents(1) - + agent = random.choice(pool) self._update_stats() self._add_to_history(agent) @@ -189,7 +189,7 @@ def browser(self, name: str) -> str: matched = self.custom(browser=name) else: matched = random.choice(matching_agents) - + self._update_stats(browser_type=name) self._add_to_history(matched) return matched @@ -248,7 +248,7 @@ def custom(self, browser: Optional[str] = None, version: Optional[str] = None, browser = browser.lower() if browser else 'chrome' v_range = BROWSERS.get(browser, (100, 130)) v_num = int(version.split('.')[0]) if version else random.randint(*v_range) - + os = os.lower() if os else random.choice(['windows', 'mac', 'linux']) os_ver = os_version or random.choice(OS_VERSIONS.get(os, ["10.0"])) device_type = (device_type or 'desktop').lower() @@ -268,7 +268,7 @@ def custom(self, browser: Optional[str] = None, version: Optional[str] = None, platform = "Windows NT 10.0; Win64; x64" agent = f"Mozilla/5.0 ({platform}) AppleWebKit/537.36 (KHTML, like Gecko) " - + if browser == 'chrome': agent += f"Chrome/{v_num}.0.0.0 Safari/537.36" elif browser == 'firefox': @@ -295,7 +295,7 @@ def generate_fingerprint(self, browser: Optional[str] = None) -> Dict[str, str]: """ ua = self.browser(browser) if browser else self.random() ip = self.rotate_ip() - + sec_ch_ua = "" for b_name in FINGERPRINTS.get("sec_ch_ua", {}): if b_name in ua.lower(): @@ -325,7 +325,7 @@ def smart_tv(self) -> str: agent = f"Mozilla/5.0 (Web0S; {tv}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36" else: agent = f"Mozilla/5.0 (Linux; {tv}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36" - + self._update_stats(device_type="tv") return agent @@ -338,7 +338,7 @@ def gaming(self) -> str: agent = f"Mozilla/5.0 (Windows NT 10.0; Win64; x64; Xbox; {console}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edge/110.0.1587.41" else: agent = self.random() - + self._update_stats(device_type="console") return agent @@ -349,7 +349,7 @@ def wearable(self) -> str: agent = "Mozilla/5.0 (AppleWatch; CPU WatchOS like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/10.0 Mobile/15E148 Safari/605.1" else: agent = f"Mozilla/5.0 (Linux; {dev}) AppleWebKit/537.36 (KHTML, like Gecko)" - + self._update_stats(device_type="wearable") return agent @@ -406,7 +406,7 @@ def rotate_proxy(self) -> Optional[str]: """Rotate through and return the next proxy from the pool.""" if not self._proxy_pool: return None - + def rot(): proxy = self._proxy_pool[self._proxy_index] self._proxy_index = (self._proxy_index + 1) % len(self._proxy_pool) diff --git a/webscout/litagent/constants.py b/webscout/litagent/constants.py index 5f8bef22..6d6c53f3 100644 --- a/webscout/litagent/constants.py +++ b/webscout/litagent/constants.py @@ -37,10 +37,10 @@ # Browser fingerprinting components FINGERPRINTS = { "accept_language": [ - "en-US,en;q=0.9", - "en-GB,en;q=0.8,en-US;q=0.6", + "en-US,en;q=0.9", + "en-GB,en;q=0.8,en-US;q=0.6", "es-ES,es;q=0.9,en;q=0.8", - "fr-FR,fr;q=0.9,en;q=0.8", + "fr-FR,fr;q=0.9,en;q=0.8", "de-DE,de;q=0.9,en;q=0.8" ], "accept": [ @@ -57,4 +57,4 @@ "platforms": [ "Windows", "macOS", "Linux", "Android", "iOS" ] -} \ No newline at end of file +} diff --git a/webscout/models.py b/webscout/models.py index 966eb4bd..d9da9a25 100644 --- a/webscout/models.py +++ b/webscout/models.py @@ -1,6 +1,7 @@ import importlib import pkgutil -from typing import Dict, List, Any, Union +from typing import Any, Dict, List, Union + from webscout.AIbase import Provider, TTSProvider # Import TTI base class @@ -15,41 +16,41 @@ class _LLMModels: """ A class for managing LLM provider models in the webscout package. """ - + def list(self) -> Dict[str, List[str]]: """ Gets all available models from each provider that has an AVAILABLE_MODELS attribute. - + Returns: Dictionary mapping provider names to their available models """ return self._get_provider_models() - + def get(self, provider_name: str) -> List[str]: """ Gets all available models for a specific provider. - + Args: provider_name: The name of the provider - + Returns: List of available models for the provider """ all_models = self._get_provider_models() return all_models.get(provider_name, []) - + def summary(self) -> Dict[str, int]: """ Returns a summary of available providers and models. - + Returns: Dictionary with provider and model counts """ provider_models = self._get_provider_models() total_providers = len(provider_models) - total_models = sum(len(models) if isinstance(models, (list, tuple, set)) + total_models = sum(len(models) if isinstance(models, (list, tuple, set)) else 1 for models in provider_models.values()) - + return { "providers": total_providers, "models": total_models, @@ -58,39 +59,39 @@ def summary(self) -> Dict[str, int]: for provider, models in provider_models.items() } } - + def providers(self) -> Dict[str, Dict[str, Any]]: """ Gets detailed information about all LLM providers including models, parameters, and metadata. - + Returns: Dictionary mapping provider names to detailed provider information """ return self._get_provider_details() - + def provider(self, provider_name: str) -> Dict[str, Any]: """ Gets detailed information about a specific LLM provider. - + Args: provider_name: The name of the provider - + Returns: Dictionary with detailed provider information """ all_providers = self._get_provider_details() return all_providers.get(provider_name, {}) - + def _get_provider_models(self) -> Dict[str, List[str]]: """ Internal method to get all available models from each provider. - + Returns: Dictionary mapping provider names to their available models """ provider_models = {} provider_package = importlib.import_module("webscout.Provider") - + for _, module_name, _ in pkgutil.iter_modules(provider_package.__path__): try: module = importlib.import_module(f"webscout.Provider.{module_name}") @@ -115,19 +116,19 @@ def _get_provider_models(self) -> Dict[str, List[str]]: provider_models[attr_name] = [] except Exception: pass - + return provider_models - + def _get_provider_details(self) -> Dict[str, Dict[str, Any]]: """ Internal method to get detailed information about all LLM providers. - + Returns: Dictionary mapping provider names to detailed provider information """ provider_details = {} provider_package = importlib.import_module("webscout.Provider") - + for _, module_name, _ in pkgutil.iter_modules(provider_package.__path__): try: module = importlib.import_module(f"webscout.Provider.{module_name}") @@ -155,21 +156,21 @@ def _get_provider_details(self) -> Dict[str, Dict[str, Any]]: models = list(available_models) else: models = [str(available_models)] - + # Sort models models = sorted(models) - + # Get supported parameters (common OpenAI-compatible parameters) supported_params = [ - "model", "messages", "max_tokens", "temperature", "top_p", + "model", "messages", "max_tokens", "temperature", "top_p", "presence_penalty", "frequency_penalty", "stop", "stream", "user" ] - + # Get additional metadata metadata = {} if hasattr(attr, '__doc__') and attr.__doc__: metadata['description'] = attr.__doc__.strip().split('\n')[0] - + provider_details[attr_name] = { "name": attr_name, "class": attr.__name__, @@ -181,50 +182,50 @@ def _get_provider_details(self) -> Dict[str, Dict[str, Any]]: } except Exception: pass - + return provider_details class _TTSModels: """ A class for managing TTS provider voices in the webscout package. """ - + def list(self) -> Dict[str, List[str]]: """ Gets all available voices from each TTS provider that has an all_voices attribute. - + Returns: Dictionary mapping TTS provider names to their available voices """ return self._get_tts_voices() - + def get(self, provider_name: str) -> Union[List[str], Dict[str, str]]: """ Gets all available voices for a specific TTS provider. - + Args: provider_name: The name of the TTS provider - + Returns: List or Dictionary of available voices for the provider """ all_voices = self._get_tts_voices() return all_voices.get(provider_name, []) - + def summary(self) -> Dict[str, Any]: """ Returns a summary of available TTS providers and voices. - + Returns: Dictionary with provider and voice counts """ provider_voices = self._get_tts_voices() total_providers = len(provider_voices) - + # Count voices, handling both list and dict formats total_voices = 0 provider_voice_counts = {} - + for provider, voices in provider_voices.items(): if isinstance(voices, dict): count = len(voices) @@ -232,29 +233,29 @@ def summary(self) -> Dict[str, Any]: count = len(voices) else: count = 1 - + total_voices += count provider_voice_counts[provider] = count - + return { "providers": total_providers, "voices": total_voices, "provider_voice_counts": provider_voice_counts } - + def _get_tts_voices(self) -> Dict[str, Union[List[str], Dict[str, str]]]: """ Internal method to get all available voices from each TTS provider. - + Returns: Dictionary mapping TTS provider names to their available voices """ provider_voices = {} - + try: # Import the TTS package specifically tts_package = importlib.import_module("webscout.Provider.TTS") - + # Iterate through TTS modules for _, module_name, _ in pkgutil.iter_modules(tts_package.__path__): try: @@ -266,74 +267,74 @@ def _get_tts_voices(self) -> Dict[str, Union[List[str], Dict[str, str]]]: if hasattr(attr, 'all_voices'): voices = attr.all_voices provider_voices[attr_name] = voices - except Exception as e: + except Exception: pass - except Exception as e: + except Exception: pass - + return provider_voices class _TTIModels: """ A class for managing TTI (Text-to-Image) provider models in the webscout package. """ - + def list(self) -> Dict[str, List[str]]: """ Gets all available models from each TTI provider that has an AVAILABLE_MODELS attribute. - + Returns: Dictionary mapping TTI provider names to their available models """ return self._get_tti_models() - + def get(self, provider_name: str) -> List[str]: """ Gets all available models for a specific TTI provider. - + Args: provider_name: The name of the TTI provider - + Returns: List of available models for the provider """ all_models = self._get_tti_models() return all_models.get(provider_name, []) - + def providers(self) -> Dict[str, Dict[str, Any]]: """ Gets detailed information about all TTI providers including models, parameters, and metadata. - + Returns: Dictionary mapping provider names to detailed provider information """ return self._get_tti_provider_details() - + def provider(self, provider_name: str) -> Dict[str, Any]: """ Gets detailed information about a specific TTI provider. - + Args: provider_name: The name of the TTI provider - + Returns: Dictionary with detailed provider information """ all_providers = self._get_tti_provider_details() return all_providers.get(provider_name, {}) - + def summary(self) -> Dict[str, int]: """ Returns a summary of available TTI providers and models. - + Returns: Dictionary with provider and model counts """ provider_models = self._get_tti_models() total_providers = len(provider_models) - total_models = sum(len(models) if isinstance(models, (list, tuple, set)) + total_models = sum(len(models) if isinstance(models, (list, tuple, set)) else 1 for models in provider_models.values()) - + return { "providers": total_providers, "models": total_models, @@ -342,26 +343,26 @@ def summary(self) -> Dict[str, int]: for provider, models in provider_models.items() } } - + def _get_tti_models(self) -> Dict[str, List[str]]: """ Internal method to get all available models from each TTI provider. - + Returns: Dictionary mapping TTI provider names to their available models """ if not TTI_AVAILABLE: return {} - + provider_models = {} tti_package = importlib.import_module("webscout.Provider.TTI") - + for _, module_name, _ in pkgutil.iter_modules(tti_package.__path__): try: module = importlib.import_module(f"webscout.Provider.TTI.{module_name}") for attr_name in dir(module): attr = getattr(module, attr_name) - if (isinstance(attr, type) and BaseImages and + if (isinstance(attr, type) and BaseImages and issubclass(attr, BaseImages) and attr != BaseImages): if hasattr(attr, 'AVAILABLE_MODELS'): # Convert any sets to lists to ensure serializability @@ -371,28 +372,28 @@ def _get_tti_models(self) -> Dict[str, List[str]]: provider_models[attr_name] = models except Exception: pass - + return provider_models - + def _get_tti_provider_details(self) -> Dict[str, Dict[str, Any]]: """ Internal method to get detailed information about all TTI providers. - + Returns: Dictionary mapping provider names to detailed provider information """ if not TTI_AVAILABLE: return {} - + provider_details = {} tti_package = importlib.import_module("webscout.Provider.TTI") - + for _, module_name, _ in pkgutil.iter_modules(tti_package.__path__): try: module = importlib.import_module(f"webscout.Provider.TTI.{module_name}") for attr_name in dir(module): attr = getattr(module, attr_name) - if (isinstance(attr, type) and BaseImages and + if (isinstance(attr, type) and BaseImages and issubclass(attr, BaseImages) and attr != BaseImages): # Get available models models = [] @@ -415,21 +416,21 @@ def _get_tti_provider_details(self) -> Dict[str, Dict[str, Any]]: models = list(available_models) else: models = [str(available_models)] - + # Sort models models = sorted(models) - + # Get supported parameters (common TTI parameters) supported_params = [ - "prompt", "model", "n", "size", "response_format", "user", + "prompt", "model", "n", "size", "response_format", "user", "style", "aspect_ratio", "timeout", "image_format", "seed" ] - + # Get additional metadata metadata = {} if hasattr(attr, '__doc__') and attr.__doc__: metadata['description'] = attr.__doc__.strip().split('\n')[0] - + provider_details[attr_name] = { "name": attr_name, "class": attr.__name__, @@ -441,7 +442,7 @@ def _get_tti_provider_details(self) -> Dict[str, Dict[str, Any]]: } except Exception: pass - + return provider_details # Create singleton instances @@ -451,10 +452,10 @@ def _get_tti_provider_details(self) -> Dict[str, Dict[str, Any]]: # Container class for all model types class Models: - def __init__(self): - self.llm = llm - self.tts = tts - self.tti = tti + def __init__(self) -> None: + self.llm: _LLMModels = llm + self.tts: _TTSModels = tts + self.tti: _TTIModels = tti # Create a singleton instance model = Models() diff --git a/webscout/optimizers.py b/webscout/optimizers.py index 5cf8e124..3fbaec1e 100644 --- a/webscout/optimizers.py +++ b/webscout/optimizers.py @@ -71,4 +71,4 @@ def coder(prompt: str) -> str: Output: """ - ) \ No newline at end of file + ) diff --git a/webscout/prompt_manager.py b/webscout/prompt_manager.py index 3e8a4538..28072d54 100644 --- a/webscout/prompt_manager.py +++ b/webscout/prompt_manager.py @@ -3,11 +3,10 @@ import json import threading -from pathlib import Path -from typing import Optional, Dict, Union -from functools import lru_cache from datetime import datetime, timedelta - +from functools import lru_cache +from pathlib import Path +from typing import Dict, Optional, Union try: from curl_cffi.requests import Session @@ -52,7 +51,7 @@ def __init__( self._cache_lock = threading.RLock() self._file_lock = threading.Lock() self._max_workers = max_workers - + self._max_workers = max_workers if CURL_AVAILABLE: self.session = Session(timeout=timeout, impersonate=impersonate) @@ -95,7 +94,7 @@ def _load_prompts(self) -> Dict[Union[str, int], str]: with self._cache_lock: if self._cache: return self._cache.copy() - + # Fallback to file if cache is empty self._load_cache() with self._cache_lock: @@ -117,45 +116,45 @@ def update_prompts_from_online(self, force: bool = False) -> bool: return True console.print("[cyan]Updating prompts...[/cyan]") - + # Fetch new prompts with timeout response = self.session.get(self.repo_url, timeout=self.timeout) response.raise_for_status() - + new_prompts = response.json() if not isinstance(new_prompts, dict): raise ValueError("Invalid response format") - + # Efficient merge with existing prompts existing_prompts = self._load_prompts() - + # Build optimized structure merged_prompts = {} string_keys = [] - + # Add existing string keys for key, value in existing_prompts.items(): if isinstance(key, str): merged_prompts[key] = value string_keys.append(key) - + # Merge new prompts (prioritize new over existing) for key, value in new_prompts.items(): if isinstance(key, str): merged_prompts[key] = value if key not in string_keys: string_keys.append(key) - + # Add numeric indices for fast access for i, key in enumerate(string_keys): merged_prompts[i] = merged_prompts[key] - + self._save_prompts(merged_prompts) self._last_update = datetime.now() - + console.print(f"[green]Updated {len([k for k in merged_prompts if isinstance(k, str)])} prompts successfully![/green]") return True - + except Exception as e: error_msg = str(e) if hasattr(e, 'response') and e.response is not None: @@ -171,7 +170,7 @@ def get_act( use_cache: bool = True ) -> Optional[str]: """Get prompt with LRU caching for performance. - + Args: key: Prompt name or index default: Default value if not found @@ -181,7 +180,7 @@ def get_act( if use_cache: return self._get_cached(key, default, case_insensitive) return self._get_uncached(key, default, case_insensitive) - + def _get_uncached( self, key: Union[str, int], @@ -191,23 +190,23 @@ def _get_uncached( """Core get logic without caching.""" with self._cache_lock: prompts = self._cache if self._cache else self._load_prompts() - + # Fast direct lookup if key in prompts: return prompts[key] - + # Case-insensitive search for string keys if isinstance(key, str) and case_insensitive: key_lower = key.lower() for k, v in prompts.items(): if isinstance(k, str) and k.lower() == key_lower: return v - + return default def add_prompt(self, name: str, prompt: str, validate: bool = True) -> bool: """Add a new prompt with validation and deduplication. - + Args: name: Name of the prompt prompt: The prompt text @@ -217,30 +216,30 @@ def add_prompt(self, name: str, prompt: str, validate: bool = True) -> bool: if not name or not prompt: console.print("[red]Name and prompt cannot be empty![/red]") return False - + if len(name) > 100 or len(prompt) > 10000: console.print("[red]Name too long (max 100) or prompt too long (max 10000)[/red]") return False - + with self._cache_lock: prompts = self._load_prompts() - + # Check for existing prompt with same content if validate: for existing_name, existing_prompt in prompts.items(): if isinstance(existing_name, str) and existing_prompt == prompt: console.print(f"[yellow]Prompt with same content exists: '{existing_name}'[/yellow]") return False - + prompts[name] = prompt - + # Update numeric indices string_keys = [k for k in prompts.keys() if isinstance(k, str)] for i, key in enumerate(string_keys): prompts[i] = prompts[key] - + self._save_prompts(prompts) - + console.print(f"[green]Added prompt: '{name}'[/green]") return True @@ -251,7 +250,7 @@ def delete_prompt( raise_not_found: bool = False ) -> bool: """Delete a prompt with proper cleanup. - + Args: name: Name or index of prompt to delete case_insensitive: Enable case-insensitive matching @@ -259,33 +258,33 @@ def delete_prompt( """ with self._cache_lock: prompts = self._load_prompts() - + # Handle direct key match if name in prompts: del prompts[name] - + # Rebuild numeric indices after deletion string_keys = [k for k in prompts.keys() if isinstance(k, str)] # Remove old numeric indices numeric_keys = [k for k in prompts.keys() if isinstance(k, int)] for key in numeric_keys: del prompts[key] - + # Add fresh numeric indices for i, key in enumerate(string_keys): prompts[i] = prompts[key] - + self._save_prompts(prompts) console.print(f"[green]Deleted prompt: '{name}'[/green]") return True - + # Handle case-insensitive match if isinstance(name, str) and case_insensitive: name_lower = name.lower() for k in list(prompts.keys()): if isinstance(k, str) and k.lower() == name_lower: return self.delete_prompt(k, case_insensitive=False, raise_not_found=raise_not_found) - + if raise_not_found: raise KeyError(f"Prompt '{name}' not found!") console.print(f"[yellow]Prompt '{name}' not found![/yellow]") @@ -297,46 +296,46 @@ def all_acts(self) -> Dict[Union[str, int], str]: with self._cache_lock: if self._cache: return self._cache.copy() - + prompts = self._load_prompts() if not prompts: self.update_prompts_from_online() prompts = self._load_prompts() - + return prompts.copy() def show_acts(self, search: Optional[str] = None, limit: int = 100) -> None: """Display prompts with optimized filtering and pagination. - + Args: search: Filter by search term limit: Maximum number of prompts to display """ prompts = self.all_acts - + # Build filtered list efficiently filtered_items = [] search_lower = search.lower() if search else None - + for key, value in prompts.items(): if isinstance(key, int): continue - + if search_lower: - if (search_lower not in key.lower() and + if (search_lower not in key.lower() and search_lower not in value.lower()): continue - + preview = value[:80] + "..." if len(value) > 80 else value filtered_items.append((str(key), preview)) - + if len(filtered_items) >= limit: break - + if not filtered_items: console.print("[yellow]No prompts found[/yellow]") return - + table = Table( title=f"Awesome Prompts ({len(filtered_items)} shown)", show_header=True, @@ -344,10 +343,10 @@ def show_acts(self, search: Optional[str] = None, limit: int = 100) -> None: ) table.add_column("Name", style="green", max_width=30) table.add_column("Preview", style="yellow", max_width=50) - + for name, preview in filtered_items: table.add_row(name, preview) - + console.print(table) def get_random_act(self) -> Optional[str]: @@ -359,4 +358,4 @@ def get_random_act(self) -> Optional[str]: import random return prompts[random.choice(string_keys)] - # End of class AwesomePrompts \ No newline at end of file + # End of class AwesomePrompts diff --git a/webscout/search/__init__.py b/webscout/search/__init__.py index 6e8c9f3e..f0d5b2d8 100644 --- a/webscout/search/__init__.py +++ b/webscout/search/__init__.py @@ -1,44 +1,43 @@ """Webscout search module - unified search interfaces.""" from .base import BaseSearch, BaseSearchEngine -from .duckduckgo_main import DuckDuckGoSearch -from .yep_main import YepSearch from .bing_main import BingSearch -from .yahoo_main import YahooSearch +from .duckduckgo_main import DuckDuckGoSearch # Import new search engines from .engines.brave import Brave from .engines.mojeek import Mojeek - -from .engines.yandex import Yandex from .engines.wikipedia import Wikipedia +from .engines.yandex import Yandex # Import result models from .results import ( - TextResult, + BooksResult, ImagesResult, - VideosResult, NewsResult, - BooksResult, + TextResult, + VideosResult, ) +from .yahoo_main import YahooSearch +from .yep_main import YepSearch __all__ = [ # Base classes "BaseSearch", "BaseSearchEngine", - + # Main search interfaces "DuckDuckGoSearch", "YepSearch", "BingSearch", "YahooSearch", - + # Individual engines "Brave", "Mojeek", "Yandex", "Wikipedia", - + # Result models "TextResult", "ImagesResult", diff --git a/webscout/search/base.py b/webscout/search/base.py index d1e2e5b7..6d663551 100644 --- a/webscout/search/base.py +++ b/webscout/search/base.py @@ -2,12 +2,13 @@ from __future__ import annotations -from litprinter import ic from abc import ABC, abstractmethod from collections.abc import Mapping from functools import cached_property from typing import Any, Generic, Literal, TypeVar +from litprinter import ic + try: from lxml import html from lxml.etree import HTMLParser as LHTMLParser @@ -41,7 +42,7 @@ class BaseSearchEngine(ABC, Generic[T]): def __init__(self, proxy: str | None = None, timeout: int | None = None, verify: bool = True): """Initialize search engine. - + Args: proxy: Proxy URL (supports http/https/socks5). timeout: Request timeout in seconds. @@ -75,14 +76,16 @@ def request(self, method: str, url: str, **kwargs: Any) -> str | None: response = self.http_client.request(method, url, **kwargs) # type: ignore return response.text except Exception as ex: - ic.configureOutput(prefix='ERROR| '); ic(f"Error in {self.name} request: {ex}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Error in {self.name} request: {ex}") return None @cached_property def parser(self) -> Any: """Get HTML parser.""" if not LXML_AVAILABLE: - ic.configureOutput(prefix='WARNING| '); ic("lxml not available, HTML parsing disabled") + ic.configureOutput(prefix='WARNING| ') + ic("lxml not available, HTML parsing disabled") return None return LHTMLParser(remove_blank_text=True, remove_comments=True, remove_pis=True, collect_ids=False) @@ -100,13 +103,13 @@ def extract_results(self, html_text: str) -> list[T]: """Extract search results from html text.""" if not LXML_AVAILABLE: raise ImportError("lxml is required for result extraction") - + html_text = self.pre_process_html(html_text) tree = self.extract_tree(html_text) - + results = [] items = tree.xpath(self.items_xpath) if self.items_xpath else [] - + for item in items: result = self.result_type() for key, xpath in self.elements_xpath.items(): @@ -117,9 +120,10 @@ def extract_results(self, html_text: str) -> list[T]: value = "".join(data) if isinstance(data, list) else data setattr(result, key, value.strip() if isinstance(value, str) else value) except Exception as ex: - ic.configureOutput(prefix='DEBUG| '); ic(f"Error extracting {key}: {ex}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Error extracting {key}: {ex}") results.append(result) - + return results def post_extract_results(self, results: list[T]) -> list[T]: @@ -154,41 +158,41 @@ class BaseSearch(ABC): """Base class for synchronous search engines (legacy).""" @abstractmethod - def text(self, *args, **kwargs) -> list[dict[str, str]]: + def text(self, *args, **kwargs) -> list[Any]: """Text search.""" raise NotImplementedError @abstractmethod - def images(self, *args, **kwargs) -> list[dict[str, str]]: + def images(self, *args, **kwargs) -> list[Any]: """Images search.""" raise NotImplementedError @abstractmethod - def videos(self, *args, **kwargs) -> list[dict[str, str]]: + def videos(self, *args, **kwargs) -> list[Any]: """Videos search.""" raise NotImplementedError @abstractmethod - def news(self, *args, **kwargs) -> list[dict[str, str]]: + def news(self, *args, **kwargs) -> list[Any]: """News search.""" raise NotImplementedError @abstractmethod - def answers(self, *args, **kwargs) -> list[dict[str, str]]: + def answers(self, *args, **kwargs) -> list[Any]: """Instant answers.""" raise NotImplementedError @abstractmethod - def suggestions(self, *args, **kwargs) -> list[dict[str, str]]: + def suggestions(self, *args, **kwargs) -> list[Any]: """Suggestions.""" raise NotImplementedError @abstractmethod - def maps(self, *args, **kwargs) -> list[dict[str, str]]: + def maps(self, *args, **kwargs) -> list[Any]: """Maps search.""" raise NotImplementedError @abstractmethod - def translate(self, *args, **kwargs) -> list[dict[str, str]]: + def translate(self, *args, **kwargs) -> list[Any]: """Translate.""" - raise NotImplementedError \ No newline at end of file + raise NotImplementedError diff --git a/webscout/search/bing_main.py b/webscout/search/bing_main.py index ac94e90b..01df4df0 100644 --- a/webscout/search/bing_main.py +++ b/webscout/search/bing_main.py @@ -1,26 +1,29 @@ """Bing unified search interface.""" from __future__ import annotations + from typing import Dict, List, Optional + from .base import BaseSearch -from .engines.bing.text import BingTextSearch from .engines.bing.images import BingImagesSearch from .engines.bing.news import BingNewsSearch from .engines.bing.suggestions import BingSuggestionsSearch +from .engines.bing.text import BingTextSearch +from .results import ImagesResult, NewsResult, TextResult class BingSearch(BaseSearch): """Unified Bing search interface.""" - def text(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None, unique: bool = True) -> List[Dict[str, str]]: + def text(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None, unique: bool = True) -> List[TextResult]: search = BingTextSearch() return search.run(keywords, region, safesearch, max_results, unique=unique) - def images(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]: + def images(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[ImagesResult]: search = BingImagesSearch() return search.run(keywords, region, safesearch, max_results) - def news(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]: + def news(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[NewsResult]: search = BingNewsSearch() return search.run(keywords, region, safesearch, max_results) @@ -39,4 +42,4 @@ def translate(self, keywords: str, from_lang: Optional[str] = None, to_lang: str raise NotImplementedError("Translate not implemented for Bing") def videos(self, *args, **kwargs) -> List[Dict[str, str]]: - raise NotImplementedError("Videos not implemented for Bing") \ No newline at end of file + raise NotImplementedError("Videos not implemented for Bing") diff --git a/webscout/search/duckduckgo_main.py b/webscout/search/duckduckgo_main.py index 0559fd84..63a7a326 100644 --- a/webscout/search/duckduckgo_main.py +++ b/webscout/search/duckduckgo_main.py @@ -1,35 +1,38 @@ """DuckDuckGo unified search interface.""" from __future__ import annotations -from typing import Dict, List, Optional + +from typing import Any, Dict, List, Optional, Union + from .base import BaseSearch -from .engines.duckduckgo.text import DuckDuckGoTextSearch +from .engines.duckduckgo.answers import DuckDuckGoAnswers from .engines.duckduckgo.images import DuckDuckGoImages -from .engines.duckduckgo.videos import DuckDuckGoVideos +from .engines.duckduckgo.maps import DuckDuckGoMaps from .engines.duckduckgo.news import DuckDuckGoNews -from .engines.duckduckgo.answers import DuckDuckGoAnswers from .engines.duckduckgo.suggestions import DuckDuckGoSuggestions -from .engines.duckduckgo.maps import DuckDuckGoMaps +from .engines.duckduckgo.text import DuckDuckGoTextSearch from .engines.duckduckgo.translate import DuckDuckGoTranslate +from .engines.duckduckgo.videos import DuckDuckGoVideos from .engines.duckduckgo.weather import DuckDuckGoWeather +from .results import ImagesResult, NewsResult, TextResult, VideosResult class DuckDuckGoSearch(BaseSearch): """Unified DuckDuckGo search interface.""" - def text(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, backend: str = "api", max_results: Optional[int] = None) -> List[Dict[str, str]]: + def text(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, backend: str = "api", max_results: Optional[int] = None) -> List[TextResult]: search = DuckDuckGoTextSearch() return search.run(keywords, region, safesearch, timelimit, backend, max_results) - def images(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, size: Optional[str] = None, color: Optional[str] = None, type_image: Optional[str] = None, layout: Optional[str] = None, license_image: Optional[str] = None, max_results: Optional[int] = None) -> List[Dict[str, str]]: + def images(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, size: Optional[str] = None, color: Optional[str] = None, type_image: Optional[str] = None, layout: Optional[str] = None, license_image: Optional[str] = None, max_results: Optional[int] = None) -> List[ImagesResult]: search = DuckDuckGoImages() return search.run(keywords, region, safesearch, timelimit, size, color, type_image, layout, license_image, max_results) - def videos(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, resolution: Optional[str] = None, duration: Optional[str] = None, license_videos: Optional[str] = None, max_results: Optional[int] = None) -> List[Dict[str, str]]: + def videos(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, resolution: Optional[str] = None, duration: Optional[str] = None, license_videos: Optional[str] = None, max_results: Optional[int] = None) -> List[VideosResult]: search = DuckDuckGoVideos() return search.run(keywords, region, safesearch, timelimit, resolution, duration, license_videos, max_results) - def news(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, max_results: Optional[int] = None) -> List[Dict[str, str]]: + def news(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, max_results: Optional[int] = None) -> List[NewsResult]: search = DuckDuckGoNews() return search.run(keywords, region, safesearch, timelimit, max_results) @@ -37,11 +40,11 @@ def answers(self, keywords: str) -> List[Dict[str, str]]: search = DuckDuckGoAnswers() return search.run(keywords) - def suggestions(self, keywords: str, region: str = "wt-wt") -> List[str]: + def suggestions(self, keywords: str, region: str = "wt-wt") -> List[Dict[str, str]]: search = DuckDuckGoSuggestions() return search.run(keywords, region) - def maps(self, keywords: str, place: Optional[str] = None, street: Optional[str] = None, city: Optional[str] = None, county: Optional[str] = None, state: Optional[str] = None, country: Optional[str] = None, postalcode: Optional[str] = None, latitude: Optional[str] = None, longitude: Optional[str] = None, radius: int = 0, max_results: Optional[int] = None) -> List[Dict[str, str]]: + def maps(self, keywords: str, place: Optional[str] = None, street: Optional[str] = None, city: Optional[str] = None, county: Optional[str] = None, state: Optional[str] = None, country: Optional[str] = None, postalcode: Optional[str] = None, latitude: Optional[str] = None, longitude: Optional[str] = None, radius: int = 0, max_results: Optional[int] = None) -> List[Dict[str, Any]]: search = DuckDuckGoMaps() return search.run(keywords, place, street, city, county, state, country, postalcode, latitude, longitude, radius, max_results) @@ -49,6 +52,6 @@ def translate(self, keywords: str, from_lang: Optional[str] = None, to_lang: str search = DuckDuckGoTranslate() return search.run(keywords, from_lang, to_lang) - def weather(self, keywords: str) -> List[Dict[str, str]]: + def weather(self, keywords: str) -> Dict[str, Any]: search = DuckDuckGoWeather() return search.run(keywords) diff --git a/webscout/search/engines/__init__.py b/webscout/search/engines/__init__.py index aae7877d..5be42820 100644 --- a/webscout/search/engines/__init__.py +++ b/webscout/search/engines/__init__.py @@ -2,33 +2,33 @@ from __future__ import annotations +from ..base import BaseSearchEngine +from .bing import BingBase, BingImagesSearch, BingNewsSearch, BingSuggestionsSearch, BingTextSearch from .brave import Brave -from .mojeek import Mojeek -from .wikipedia import Wikipedia -from .yandex import Yandex -from .bing import BingBase, BingTextSearch, BingImagesSearch, BingNewsSearch, BingSuggestionsSearch from .duckduckgo import ( + DuckDuckGoAnswers, DuckDuckGoBase, - DuckDuckGoTextSearch, DuckDuckGoImages, - DuckDuckGoVideos, + DuckDuckGoMaps, DuckDuckGoNews, - DuckDuckGoAnswers, DuckDuckGoSuggestions, - DuckDuckGoMaps, + DuckDuckGoTextSearch, DuckDuckGoTranslate, + DuckDuckGoVideos, DuckDuckGoWeather, ) -from .yep import YepBase, YepTextSearch, YepImages, YepSuggestions +from .mojeek import Mojeek +from .wikipedia import Wikipedia from .yahoo import ( - YahooSearchEngine, - YahooText, YahooImages, - YahooVideos, YahooNews, + YahooSearchEngine, YahooSuggestions, + YahooText, + YahooVideos, ) -from ..base import BaseSearchEngine +from .yandex import Yandex +from .yep import YepBase, YepImages, YepSuggestions, YepTextSearch # Engine categories mapping ENGINES = { diff --git a/webscout/search/engines/bing/__init__.py b/webscout/search/engines/bing/__init__.py index fe7980c5..6447e62f 100644 --- a/webscout/search/engines/bing/__init__.py +++ b/webscout/search/engines/bing/__init__.py @@ -12,4 +12,4 @@ "BingImagesSearch", "BingNewsSearch", "BingSuggestionsSearch", -] \ No newline at end of file +] diff --git a/webscout/search/engines/bing/base.py b/webscout/search/engines/bing/base.py index 75dcf6e5..0caaca41 100644 --- a/webscout/search/engines/bing/base.py +++ b/webscout/search/engines/bing/base.py @@ -2,9 +2,10 @@ from __future__ import annotations -from ....litagent import LitAgent from curl_cffi.requests import Session +from ....litagent import LitAgent + class BingBase: """Base class for Bing search engines.""" @@ -30,4 +31,4 @@ def __init__( timeout=timeout, impersonate=impersonate, ) - self.session.headers.update(LitAgent().generate_fingerprint()) \ No newline at end of file + self.session.headers.update(LitAgent().generate_fingerprint()) diff --git a/webscout/search/engines/bing/images.py b/webscout/search/engines/bing/images.py index 6aa15cdc..8004e265 100644 --- a/webscout/search/engines/bing/images.py +++ b/webscout/search/engines/bing/images.py @@ -2,21 +2,22 @@ from __future__ import annotations +from time import sleep from typing import List from urllib.parse import urlencode -from time import sleep -from .base import BingBase from webscout.scout import Scout from webscout.search.results import ImagesResult +from .base import BingBase + class BingImagesSearch(BingBase): name = "bing" category = "images" def run(self, *args, **kwargs) -> List[ImagesResult]: keywords = args[0] if args else kwargs.get("keywords") - region = args[1] if len(args) > 1 else kwargs.get("region", "us") + args[1] if len(args) > 1 else kwargs.get("region", "us") safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate") max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10) @@ -31,7 +32,7 @@ def run(self, *args, **kwargs) -> List[ImagesResult]: "moderate": "Moderate", "off": "Off" } - safe = safe_map.get(safesearch.lower(), "Moderate") + safe_map.get(safesearch.lower(), "Moderate") # Bing images URL url = f"{self.base_url}/images/async" @@ -85,7 +86,7 @@ def run(self, *args, **kwargs) -> List[ImagesResult]: m_data = json.loads(m_attr) image_url = m_data.get('murl', src) thumbnail = m_data.get('turl', src) - except: + except Exception: pass source = '' @@ -110,4 +111,4 @@ def run(self, *args, **kwargs) -> List[ImagesResult]: if self.sleep_interval: sleep(self.sleep_interval) - return results[:max_results] \ No newline at end of file + return results[:max_results] diff --git a/webscout/search/engines/bing/news.py b/webscout/search/engines/bing/news.py index 40278e91..4e281a35 100644 --- a/webscout/search/engines/bing/news.py +++ b/webscout/search/engines/bing/news.py @@ -2,14 +2,15 @@ from __future__ import annotations +from time import sleep from typing import List from urllib.parse import urlencode -from time import sleep -from .base import BingBase from webscout.scout import Scout from webscout.search.results import NewsResult +from .base import BingBase + class BingNewsSearch(BingBase): name = "bing" @@ -31,7 +32,7 @@ def run(self, *args, **kwargs) -> List[NewsResult]: "moderate": "Moderate", "off": "Off" } - safe = safe_map.get(safesearch.lower(), "Moderate") + safe_map.get(safesearch.lower(), "Moderate") # Bing news URL url = f"{self.base_url}/news/infinitescrollajax" @@ -92,4 +93,4 @@ def run(self, *args, **kwargs) -> List[NewsResult]: if self.sleep_interval: sleep(self.sleep_interval) - return results[:max_results] \ No newline at end of file + return results[:max_results] diff --git a/webscout/search/engines/bing/suggestions.py b/webscout/search/engines/bing/suggestions.py index 4126bf75..11f83744 100644 --- a/webscout/search/engines/bing/suggestions.py +++ b/webscout/search/engines/bing/suggestions.py @@ -33,4 +33,4 @@ def run(self, *args, **kwargs) -> List[str]: return data[1] return [] except Exception as e: - raise Exception(f"Failed to fetch suggestions: {str(e)}") \ No newline at end of file + raise Exception(f"Failed to fetch suggestions: {str(e)}") diff --git a/webscout/search/engines/bing/text.py b/webscout/search/engines/bing/text.py index 96fc6538..68fccc6f 100644 --- a/webscout/search/engines/bing/text.py +++ b/webscout/search/engines/bing/text.py @@ -2,21 +2,21 @@ from __future__ import annotations -from typing import List -from urllib.parse import urlencode from time import sleep +from typing import List -from .base import BingBase from webscout.scout import Scout from webscout.search.results import TextResult +from .base import BingBase + class BingTextSearch(BingBase): name = "bing" category = "text" def run(self, *args, **kwargs) -> List[TextResult]: keywords = args[0] if args else kwargs.get("keywords") - region = args[1] if len(args) > 1 else kwargs.get("region", "us") + args[1] if len(args) > 1 else kwargs.get("region", "us") safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate") max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10) unique = kwargs.get("unique", True) @@ -32,7 +32,7 @@ def run(self, *args, **kwargs) -> List[TextResult]: "moderate": "Moderate", "off": "Off" } - safe = safe_map.get(safesearch.lower(), "Moderate") + safe_map.get(safesearch.lower(), "Moderate") fetched_results = [] fetched_links = set() @@ -84,7 +84,7 @@ def fetch_page(url): import base64 decoded = base64.urlsafe_b64decode(encoded_url).decode() href = decoded - except: + except Exception: pass if unique and href in fetched_links: @@ -106,4 +106,4 @@ def fetch_page(url): if self.sleep_interval: sleep(self.sleep_interval) - return fetched_results[:max_results] \ No newline at end of file + return fetched_results[:max_results] diff --git a/webscout/search/engines/brave.py b/webscout/search/engines/brave.py index 6ff6235b..24f93f1b 100644 --- a/webscout/search/engines/brave.py +++ b/webscout/search/engines/brave.py @@ -44,13 +44,13 @@ def build_payload( def run(self, *args, **kwargs) -> list[TextResult]: """Run text search on Brave. - + Args: keywords: Search query. region: Region code. safesearch: Safe search level. max_results: Maximum number of results (ignored for now). - + Returns: List of TextResult objects. """ @@ -58,7 +58,7 @@ def run(self, *args, **kwargs) -> list[TextResult]: region = args[1] if len(args) > 1 else kwargs.get("region", "us-en") safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate") max_results = args[3] if len(args) > 3 else kwargs.get("max_results") - + results = self.search(query=keywords, region=region, safesearch=safesearch) if results and max_results: results = results[:max_results] diff --git a/webscout/search/engines/duckduckgo/answers.py b/webscout/search/engines/duckduckgo/answers.py index 79227921..110cd955 100644 --- a/webscout/search/engines/duckduckgo/answers.py +++ b/webscout/search/engines/duckduckgo/answers.py @@ -2,22 +2,21 @@ from __future__ import annotations -from ....exceptions import WebscoutE from .base import DuckDuckGoBase class DuckDuckGoAnswers(DuckDuckGoBase): """DuckDuckGo instant answers.""" - + name = "duckduckgo" category = "answers" - + def run(self, *args, **kwargs) -> list[dict[str, str]]: """Get instant answers from DuckDuckGo. - + Args: keywords: Search query. - + Returns: List of answer dictionaries. """ diff --git a/webscout/search/engines/duckduckgo/base.py b/webscout/search/engines/duckduckgo/base.py index adb301a5..2fd3d002 100644 --- a/webscout/search/engines/duckduckgo/base.py +++ b/webscout/search/engines/duckduckgo/base.py @@ -11,7 +11,7 @@ from typing import Any try: - import trio # type: ignore + import trio # type: ignore except ImportError: pass @@ -25,13 +25,13 @@ LXML_AVAILABLE = False from ....exceptions import RatelimitE, TimeoutE, WebscoutE +from ....litagent import LitAgent from ....utils import ( _extract_vqd, _normalize, _normalize_url, json_loads, ) -from ....litagent import LitAgent class DuckDuckGoBase: @@ -66,7 +66,7 @@ def __init__( """ ddgs_proxy: str | None = os.environ.get("DDGS_PROXY") self.proxy: str | None = ddgs_proxy if ddgs_proxy else proxy - + if not proxy and proxies: self.proxy = proxies.get("http") or proxies.get("https") if isinstance(proxies, dict) else proxies @@ -89,7 +89,7 @@ def __init__( ) self.timeout = timeout self.sleep_timestamp = 0.0 - + # Utility methods self.cycle = cycle self.islice = islice @@ -99,7 +99,7 @@ def parser(self) -> Any: """Get HTML parser.""" if not LXML_AVAILABLE: raise ImportError("lxml is required for HTML parsing") - + class Parser: def __init__(self): self.lhtml_parser = LHTMLParser( @@ -109,10 +109,10 @@ def __init__(self): collect_ids=False ) self.etree = __import__('lxml.etree', fromlist=['Element']) - + def fromstring(self, html: bytes | str) -> Any: return document_fromstring(html, parser=self.lhtml_parser) - + return Parser() def _sleep(self, sleeptime: float = 0.75) -> None: @@ -162,7 +162,7 @@ def _get_url( if "time" in str(ex).lower(): raise TimeoutE(f"{url} {type(ex).__name__}: {ex}") from ex raise WebscoutE(f"{url} {type(ex).__name__}: {ex}") from ex - + if resp.status_code == 200: return resp elif resp.status_code in (202, 301, 403, 400, 429, 418): diff --git a/webscout/search/engines/duckduckgo/images.py b/webscout/search/engines/duckduckgo/images.py index 27de59fe..f6d1f9a7 100644 --- a/webscout/search/engines/duckduckgo/images.py +++ b/webscout/search/engines/duckduckgo/images.py @@ -2,20 +2,19 @@ from __future__ import annotations -from ....exceptions import WebscoutE from ....search.results import ImagesResult from .base import DuckDuckGoBase class DuckDuckGoImages(DuckDuckGoBase): """DuckDuckGo image search.""" - + name = "duckduckgo" category = "images" - + def run(self, *args, **kwargs) -> list[ImagesResult]: """Perform image search on DuckDuckGo. - + Args: keywords: Search query. region: Region code. @@ -27,7 +26,7 @@ def run(self, *args, **kwargs) -> list[ImagesResult]: layout: Square, Tall, Wide. license_image: any, Public, Share, etc. max_results: Maximum number of results. - + Returns: List of ImagesResult objects. """ diff --git a/webscout/search/engines/duckduckgo/maps.py b/webscout/search/engines/duckduckgo/maps.py index 12531a83..e288cc8b 100644 --- a/webscout/search/engines/duckduckgo/maps.py +++ b/webscout/search/engines/duckduckgo/maps.py @@ -17,7 +17,7 @@ def _calculate_distance(self, lat_t: Decimal, lon_l: Decimal, lat_b: Decimal, lo lon_l_f = float(lon_l) lat_b_f = float(lat_b) lon_r_f = float(lon_r) - + # Calculate Euclidean distance distance = sqrt((lat_t_f - lat_b_f) ** 2 + (lon_r_f - lon_l_f) ** 2) return distance diff --git a/webscout/search/engines/duckduckgo/news.py b/webscout/search/engines/duckduckgo/news.py index 2a8ac544..5307e050 100644 --- a/webscout/search/engines/duckduckgo/news.py +++ b/webscout/search/engines/duckduckgo/news.py @@ -2,7 +2,6 @@ from datetime import datetime, timezone -from ....exceptions import WebscoutE from ....search.results import NewsResult from .base import DuckDuckGoBase diff --git a/webscout/search/engines/duckduckgo/suggestions.py b/webscout/search/engines/duckduckgo/suggestions.py index 607eba8c..4d12b6fb 100644 --- a/webscout/search/engines/duckduckgo/suggestions.py +++ b/webscout/search/engines/duckduckgo/suggestions.py @@ -1,6 +1,5 @@ from __future__ import annotations -from ....exceptions import WebscoutE from .base import DuckDuckGoBase diff --git a/webscout/search/engines/duckduckgo/text.py b/webscout/search/engines/duckduckgo/text.py index e3b2fc9c..8e23374a 100644 --- a/webscout/search/engines/duckduckgo/text.py +++ b/webscout/search/engines/duckduckgo/text.py @@ -12,13 +12,13 @@ class DuckDuckGoTextSearch(DuckDuckGoBase): """DuckDuckGo text/web search.""" - + name = "duckduckgo" category = "text" - + def run(self, *args, **kwargs) -> list[TextResult]: """Perform text search on DuckDuckGo. - + Args: keywords: Search query. region: Region code (e.g., wt-wt, us-en). @@ -26,13 +26,13 @@ def run(self, *args, **kwargs) -> list[TextResult]: timelimit: d, w, m, or y. backend: html, lite, or auto. max_results: Maximum number of results. - + Returns: List of TextResult objects. """ keywords = args[0] if args else kwargs.get("keywords") region = args[1] if len(args) > 1 else kwargs.get("region", "wt-wt") - safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate") + args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate") timelimit = args[3] if len(args) > 3 else kwargs.get("timelimit") backend = args[4] if len(args) > 4 else kwargs.get("backend", "auto") max_results = args[5] if len(args) > 5 else kwargs.get("max_results") diff --git a/webscout/search/engines/duckduckgo/translate.py b/webscout/search/engines/duckduckgo/translate.py index d4c827e7..e0401a51 100644 --- a/webscout/search/engines/duckduckgo/translate.py +++ b/webscout/search/engines/duckduckgo/translate.py @@ -1,6 +1,5 @@ from __future__ import annotations -from ....exceptions import WebscoutE from .base import DuckDuckGoBase diff --git a/webscout/search/engines/duckduckgo/videos.py b/webscout/search/engines/duckduckgo/videos.py index 211227c2..99c23343 100644 --- a/webscout/search/engines/duckduckgo/videos.py +++ b/webscout/search/engines/duckduckgo/videos.py @@ -1,6 +1,5 @@ from __future__ import annotations -from ....exceptions import WebscoutE from ....search.results import VideosResult from .base import DuckDuckGoBase diff --git a/webscout/search/engines/mojeek.py b/webscout/search/engines/mojeek.py index 8c9c9259..fde1d694 100644 --- a/webscout/search/engines/mojeek.py +++ b/webscout/search/engines/mojeek.py @@ -38,13 +38,13 @@ def build_payload( def run(self, *args, **kwargs) -> list[TextResult]: """Run text search on Mojeek. - + Args: keywords: Search query. region: Region code. safesearch: Safe search level. max_results: Maximum number of results (ignored for now). - + Returns: List of TextResult objects. """ @@ -52,7 +52,7 @@ def run(self, *args, **kwargs) -> list[TextResult]: region = args[1] if len(args) > 1 else kwargs.get("region", "us-en") safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate") max_results = args[3] if len(args) > 3 else kwargs.get("max_results") - + results = self.search(query=keywords, region=region, safesearch=safesearch) if results and max_results: results = results[:max_results] diff --git a/webscout/search/engines/wikipedia.py b/webscout/search/engines/wikipedia.py index 22d496e8..38bcd003 100644 --- a/webscout/search/engines/wikipedia.py +++ b/webscout/search/engines/wikipedia.py @@ -2,13 +2,12 @@ from __future__ import annotations -from litprinter import ic from typing import Any from urllib.parse import quote +from ...utils import json_loads from ..base import BaseSearchEngine from ..results import TextResult -from ...utils import json_loads class Wikipedia(BaseSearchEngine[TextResult]): @@ -40,28 +39,28 @@ def extract_results(self, html_text: str) -> list[TextResult]: json_data = json_loads(html_text) if not json_data or len(json_data) < 4: return [] - + results = [] titles, descriptions, urls = json_data[1], json_data[2], json_data[3] - + for title, description, url in zip(titles, descriptions, urls): result = TextResult() result.title = title result.body = description result.href = url results.append(result) - + return results def run(self, *args, **kwargs) -> list[TextResult]: """Run text search on Wikipedia. - + Args: keywords: Search query. region: Region code. safesearch: Safe search level (ignored). max_results: Maximum number of results. - + Returns: List of TextResult objects. """ @@ -69,7 +68,7 @@ def run(self, *args, **kwargs) -> list[TextResult]: region = args[1] if len(args) > 1 else kwargs.get("region", "en-us") safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate") max_results = args[3] if len(args) > 3 else kwargs.get("max_results") - + results = self.search(query=keywords, region=region, safesearch=safesearch) if results and max_results: results = results[:max_results] diff --git a/webscout/search/engines/yahoo/__init__.py b/webscout/search/engines/yahoo/__init__.py index aed2f9d8..2212c08b 100644 --- a/webscout/search/engines/yahoo/__init__.py +++ b/webscout/search/engines/yahoo/__init__.py @@ -15,11 +15,11 @@ Example: >>> from webscout.search.engines.yahoo import YahooText - >>> + >>> >>> # Search with automatic pagination >>> searcher = YahooText() >>> results = searcher.search("python programming", max_results=50) - >>> + >>> >>> for result in results: ... print(f"{result.title}: {result.url}") """ diff --git a/webscout/search/engines/yahoo/answers.py b/webscout/search/engines/yahoo/answers.py index 0d429e5b..ade1c576 100644 --- a/webscout/search/engines/yahoo/answers.py +++ b/webscout/search/engines/yahoo/answers.py @@ -13,4 +13,4 @@ def run(self, *args, **kwargs) -> list[dict[str, str]]: Not supported. """ - raise NotImplementedError("Yahoo does not support instant answers") \ No newline at end of file + raise NotImplementedError("Yahoo does not support instant answers") diff --git a/webscout/search/engines/yahoo/base.py b/webscout/search/engines/yahoo/base.py index a9342ddf..10d7dac0 100644 --- a/webscout/search/engines/yahoo/base.py +++ b/webscout/search/engines/yahoo/base.py @@ -3,7 +3,7 @@ from __future__ import annotations from secrets import token_urlsafe -from typing import Any, Generic, TypeVar +from typing import Generic, TypeVar from ...base import BaseSearchEngine @@ -11,22 +11,22 @@ class YahooSearchEngine(BaseSearchEngine[T], Generic[T]): """Base class for Yahoo search engines. - + Yahoo search is powered by Bing but has its own interface. All Yahoo searches use dynamic URLs with tokens for tracking. """ provider = "yahoo" _base_url = "https://search.yahoo.com" - + def generate_ylt_token(self) -> str: """Generate Yahoo _ylt tracking token.""" return token_urlsafe(24 * 3 // 4) - + def generate_ylu_token(self) -> str: """Generate Yahoo _ylu tracking token.""" return token_urlsafe(47 * 3 // 4) - + def build_search_url(self, base_path: str) -> str: """Build search URL with tracking tokens.""" ylt = self.generate_ylt_token() diff --git a/webscout/search/engines/yahoo/images.py b/webscout/search/engines/yahoo/images.py index 9403626d..10fcbeb1 100644 --- a/webscout/search/engines/yahoo/images.py +++ b/webscout/search/engines/yahoo/images.py @@ -4,15 +4,14 @@ from collections.abc import Mapping from typing import Any -from urllib.parse import urljoin -from .base import YahooSearchEngine from ...results import ImagesResult +from .base import YahooSearchEngine class YahooImages(YahooSearchEngine[ImagesResult]): """Yahoo image search engine with filter support. - + Features: - Size filters (small, medium, large, wallpaper) - Color filters (color, bw, red, orange, yellow, etc.) @@ -20,7 +19,7 @@ class YahooImages(YahooSearchEngine[ImagesResult]): - Layout filters (square, wide, tall) - Time filters - Pagination support - + Note: Yahoo does not support reverse image search (searching by image upload/URL). For reverse image search functionality, use Google Images or Bing Images instead. """ @@ -38,7 +37,7 @@ class YahooImages(YahooSearchEngine[ImagesResult]): items_xpath = "//li[contains(@class, 'ld')]" elements_xpath: Mapping[str, str] = { "title": "@data", - "image": "@data", + "image": "@data", "thumbnail": "@data", "url": "@data", "source": "@data", @@ -49,7 +48,7 @@ class YahooImages(YahooSearchEngine[ImagesResult]): # Filter mappings SIZE_FILTERS = { "small": "small", - "medium": "medium", + "medium": "medium", "large": "large", "wallpaper": "wallpaper", "all": "", @@ -99,7 +98,7 @@ def build_payload( **kwargs: Any, ) -> dict[str, Any]: """Build image search payload with filters. - + Args: query: Search query region: Region code @@ -112,7 +111,7 @@ def build_payload( - type: Image type filter - layout: Layout/aspect ratio filter - license: Usage rights filter - + Returns: Query parameters dictionary """ @@ -169,40 +168,40 @@ def build_payload( def post_extract_results(self, results: list[ImagesResult]) -> list[ImagesResult]: """Post-process image results to parse JSON data. - + Args: results: Raw extracted results - + Returns: Cleaned results with proper URLs and metadata """ import json from urllib.parse import unquote - + cleaned_results = [] - + for result in results: # Parse JSON data from the data attribute if result.title and result.title.startswith('{'): try: data = json.loads(result.title) - + # Extract title result.title = data.get('desc', '') or data.get('tit', '') - + # Extract URLs result.url = data.get('rurl', '') result.thumbnail = data.get('turl', '') result.image = data.get('turlL', '') or data.get('turl', '') - + # Extract dimensions result.width = int(data.get('imgW', 0)) result.height = int(data.get('imgH', 0)) - + except (json.JSONDecodeError, KeyError, ValueError): # If JSON parsing fails, keep original data pass - + # Clean URLs if they exist if result.url: result.url = unquote(result.url) @@ -210,9 +209,9 @@ def post_extract_results(self, results: list[ImagesResult]) -> list[ImagesResult result.image = unquote(result.image) if result.thumbnail: result.thumbnail = unquote(result.thumbnail) - + cleaned_results.append(result) - + return cleaned_results def search( @@ -226,7 +225,7 @@ def search( **kwargs: Any, ) -> list[ImagesResult] | None: """Search Yahoo Images with pagination. - + Args: query: Image search query region: Region code @@ -235,14 +234,14 @@ def search( page: Starting page max_results: Maximum results to return **kwargs: Additional filters (size, color, type, layout) - + Returns: List of ImageResult objects """ results = [] current_page = page max_pages = kwargs.get("max_pages", 5) - + while current_page <= max_pages: payload = self.build_payload( query=query, @@ -252,29 +251,29 @@ def search( page=current_page, **kwargs ) - + html_text = self.request(self.search_method, self.search_url, params=payload) if not html_text: break - + html_text = self.pre_process_html(html_text) page_results = self.extract_results(html_text) - + if not page_results: break - + results.extend(page_results) - + if max_results and len(results) >= max_results: break - + current_page += 1 - + results = self.post_extract_results(results) - + if max_results: results = results[:max_results] - + return results if results else None def run( @@ -291,7 +290,7 @@ def run( max_results: int | None = None, ) -> list[dict[str, str]]: """Run image search and return results as dictionaries. - + Args: keywords: Search query. region: Region code. @@ -303,7 +302,7 @@ def run( layout: Layout filter. license_image: License filter. max_results: Maximum number of results. - + Returns: List of image result dictionaries. """ diff --git a/webscout/search/engines/yahoo/maps.py b/webscout/search/engines/yahoo/maps.py index 5aa28c5b..cf611464 100644 --- a/webscout/search/engines/yahoo/maps.py +++ b/webscout/search/engines/yahoo/maps.py @@ -13,4 +13,4 @@ def run(self, *args, **kwargs) -> list[dict[str, str]]: Not supported. """ - raise NotImplementedError("Yahoo does not support maps search") \ No newline at end of file + raise NotImplementedError("Yahoo does not support maps search") diff --git a/webscout/search/engines/yahoo/news.py b/webscout/search/engines/yahoo/news.py index 4b1068bc..7fa34a95 100644 --- a/webscout/search/engines/yahoo/news.py +++ b/webscout/search/engines/yahoo/news.py @@ -6,47 +6,47 @@ from secrets import token_urlsafe from typing import Any -from .base import YahooSearchEngine from ...results import NewsResult +from .base import YahooSearchEngine def extract_image(u: str) -> str: """Sanitize image URL. - + Args: u: Image URL - + Returns: Cleaned URL or empty string """ if not u: return "" - + # Skip data URIs if u.startswith("data:image"): return "" - + return u def extract_source(s: str) -> str: """Remove ' via Yahoo' from source string. - + Args: s: Source string - + Returns: Cleaned source name """ if not s: return s - + return s.replace(" via Yahoo", "").replace(" - Yahoo", "").strip() class YahooNews(YahooSearchEngine[NewsResult]): """Yahoo news search engine with advanced filtering. - + Features: - Time-based filtering - Category filtering @@ -82,7 +82,7 @@ def build_payload( **kwargs: Any, ) -> dict[str, Any]: """Build news search payload. - + Args: query: Search query region: Region code @@ -90,7 +90,7 @@ def build_payload( timelimit: Time filter (d, w, m) page: Page number **kwargs: Additional parameters - + Returns: Query parameters dictionary """ @@ -100,17 +100,17 @@ def build_payload( f";_ylt={token_urlsafe(24 * 3 // 4)}" f";_ylu={token_urlsafe(47 * 3 // 4)}" ) - + payload = { "p": query, "ei": "UTF-8", } - + # Pagination - Yahoo news uses 'b' parameter if page > 1: # Each page shows approximately 10 articles payload["b"] = f"{(page - 1) * 10 + 1}" - + # Time filter if timelimit: time_map = { @@ -120,35 +120,35 @@ def build_payload( } if timelimit in time_map: payload["btf"] = time_map[timelimit] - + # Additional filters if "category" in kwargs: payload["category"] = kwargs["category"] - + if "sort" in kwargs: # Sort by relevance or date payload["sort"] = kwargs["sort"] - + return payload def post_extract_results(self, results: list[NewsResult]) -> list[NewsResult]: """Post-process news results. - + Args: results: Raw extracted results - + Returns: Cleaned news results """ cleaned_results = [] - + for result in results: # Clean image URL result.image = extract_image(result.image) - + # Clean source name result.source = extract_source(result.source) - + # Extract URL from redirect if result.url and "/RU=" in result.url: from urllib.parse import unquote @@ -157,11 +157,11 @@ def post_extract_results(self, results: list[NewsResult]) -> list[NewsResult]: if end == -1: end = len(result.url) result.url = unquote(result.url[start:end]) - + # Filter out results without essential fields if result.title and result.url: cleaned_results.append(result) - + return cleaned_results def search( @@ -175,7 +175,7 @@ def search( **kwargs: Any, ) -> list[NewsResult] | None: """Search Yahoo News with pagination. - + Args: query: News search query region: Region code @@ -184,14 +184,14 @@ def search( page: Starting page max_results: Maximum results to return **kwargs: Additional parameters (category, sort) - + Returns: List of NewsResult objects """ results = [] current_page = page max_pages = kwargs.get("max_pages", 10) - + while current_page <= max_pages: payload = self.build_payload( query=query, @@ -201,29 +201,29 @@ def search( page=current_page, **kwargs ) - + html_text = self.request(self.search_method, self.search_url, params=payload) if not html_text: break - + html_text = self.pre_process_html(html_text) page_results = self.extract_results(html_text) - + if not page_results: break - + results.extend(page_results) - + if max_results and len(results) >= max_results: break - + current_page += 1 - + results = self.post_extract_results(results) - + if max_results: results = results[:max_results] - + return results if results else None def run( @@ -235,14 +235,14 @@ def run( max_results: int | None = None, ) -> list[dict[str, str]]: """Run news search and return results as dictionaries. - + Args: keywords: Search query. region: Region code. safesearch: Safe search level. timelimit: Time filter. max_results: Maximum number of results. - + Returns: List of news result dictionaries. """ diff --git a/webscout/search/engines/yahoo/suggestions.py b/webscout/search/engines/yahoo/suggestions.py index 42c5b3ea..905d32f2 100644 --- a/webscout/search/engines/yahoo/suggestions.py +++ b/webscout/search/engines/yahoo/suggestions.py @@ -10,7 +10,7 @@ class YahooSuggestions(YahooSearchEngine[str]): """Yahoo search suggestions engine. - + Provides autocomplete suggestions as you type. """ @@ -30,7 +30,7 @@ def build_payload( **kwargs: Any, ) -> dict[str, Any]: """Build suggestions payload. - + Args: query: Partial search query region: Region code @@ -38,7 +38,7 @@ def build_payload( timelimit: Time limit (unused) page: Page number (unused) **kwargs: Additional parameters - + Returns: Query parameters """ @@ -47,21 +47,21 @@ def build_payload( "output": "sd1", "nresults": kwargs.get("max_suggestions", 10), } - + return payload def extract_results(self, html_text: str) -> list[str]: """Extract suggestions from JSON response. - + Args: html_text: JSON response text - + Returns: List of suggestion strings """ try: data = json.loads(html_text) - + # Yahoo returns suggestions in 'r' key if "r" in data and isinstance(data["r"], list): suggestions = [] @@ -71,7 +71,7 @@ def extract_results(self, html_text: str) -> list[str]: elif isinstance(item, str): suggestions.append(item) return suggestions - + return [] except (json.JSONDecodeError, KeyError, TypeError): return [] @@ -87,7 +87,7 @@ def search( **kwargs: Any, ) -> list[str] | None: """Get search suggestions for a query. - + Args: query: Partial search query region: Region code @@ -96,13 +96,13 @@ def search( page: Page number max_results: Maximum suggestions **kwargs: Additional parameters - + Returns: List of suggestion strings """ if max_results: kwargs["max_suggestions"] = max_results - + payload = self.build_payload( query=query, region=region, @@ -111,25 +111,25 @@ def search( page=page, **kwargs ) - + response = self.request(self.search_method, self.search_url, params=payload) if not response: return None - + suggestions = self.extract_results(response) - + if max_results: suggestions = suggestions[:max_results] - + return suggestions if suggestions else None def run(self, keywords: str, region: str = "us-en") -> list[str]: """Run suggestions search and return results. - + Args: keywords: Search query. region: Region code. - + Returns: List of suggestion strings. """ diff --git a/webscout/search/engines/yahoo/text.py b/webscout/search/engines/yahoo/text.py index 3ccab5fa..a7c1f22a 100644 --- a/webscout/search/engines/yahoo/text.py +++ b/webscout/search/engines/yahoo/text.py @@ -4,21 +4,21 @@ from collections.abc import Mapping from typing import Any -from urllib.parse import unquote_plus, urljoin +from urllib.parse import unquote_plus -from .base import YahooSearchEngine from ...results import TextResult +from .base import YahooSearchEngine def extract_url(u: str) -> str: """Extract and sanitize URL from Yahoo redirect. - + Yahoo uses /RU= redirect URLs that need to be decoded. Example: /url?sa=t&url=https%3A%2F%2Fexample.com """ if not u: return u - + # Handle /RU= redirect format if "/RU=" in u: start = u.find("/RU=") + 4 @@ -26,13 +26,13 @@ def extract_url(u: str) -> str: if end == -1: end = len(u) return unquote_plus(u[start:end]) - + return u class YahooText(YahooSearchEngine[TextResult]): """Yahoo text search engine with full pagination support. - + Features: - Multi-page navigation like a human - Automatic next page detection @@ -56,16 +56,16 @@ class YahooText(YahooSearchEngine[TextResult]): } def build_payload( - self, - query: str, - region: str, - safesearch: str, - timelimit: str | None, - page: int = 1, + self, + query: str, + region: str, + safesearch: str, + timelimit: str | None, + page: int = 1, **kwargs: Any ) -> dict[str, Any]: """Build search payload for Yahoo. - + Args: query: Search query string region: Region code (e.g., 'us-en') @@ -73,7 +73,7 @@ def build_payload( timelimit: Time limit filter (d=day, w=week, m=month) page: Page number (1-indexed) **kwargs: Additional parameters - + Returns: Dictionary of query parameters """ @@ -81,40 +81,40 @@ def build_payload( "p": query, "ei": "UTF-8", } - + # Pagination: Yahoo uses 'b' parameter for offset # Page 1: no b parameter or b=1 # Page 2: b=8 (shows results 8-14) # Page 3: b=15, etc. if page > 1: payload["b"] = f"{(page - 1) * 7 + 1}" - + # Time filter if timelimit: payload["btf"] = timelimit - + return payload def post_extract_results(self, results: list[TextResult]) -> list[TextResult]: """Post-process and clean extracted results. - + Args: results: Raw extracted results - + Returns: Cleaned and filtered results """ cleaned_results = [] - + for result in results: # Extract real URL from redirect if result.href: result.href = extract_url(result.href) - + # Filter out empty results if result.title and result.href: cleaned_results.append(result) - + return cleaned_results def search( @@ -128,10 +128,10 @@ def search( **kwargs: Any, ) -> list[TextResult] | None: """Search Yahoo with automatic pagination like a human browser. - + This method automatically follows pagination links to gather results across multiple pages, similar to how a human would browse search results. - + Args: query: Search query string region: Region code @@ -140,14 +140,14 @@ def search( page: Starting page number max_results: Maximum number of results to return **kwargs: Additional search parameters - + Returns: List of TextResult objects, or None if search fails """ results = [] current_page = page max_pages = kwargs.get("max_pages", 10) # Limit to prevent infinite loops - + while current_page <= max_pages: # Build payload for current page payload = self.build_payload( @@ -158,45 +158,45 @@ def search( page=current_page, **kwargs ) - + # Make request html_text = self.request(self.search_method, self.search_url, params=payload) if not html_text: break - + # Pre-process HTML html_text = self.pre_process_html(html_text) - + # Extract results from current page page_results = self.extract_results(html_text) if not page_results: break - + results.extend(page_results) - + # Check if we have enough results if max_results and len(results) >= max_results: break - + # Look for next page link tree = self.extract_tree(html_text) next_links = tree.xpath("//a[contains(text(), 'Next') or contains(@class, 'next')]/@href") - + if not next_links: # Try to find numbered page links page_links = tree.xpath(f"//a[contains(text(), '{current_page + 1}')]/@href") if not page_links: break - + current_page += 1 - + # Post-process all results results = self.post_extract_results(results) - + # Trim to max_results if specified if max_results: results = results[:max_results] - + return results if results else None def search_page( @@ -209,7 +209,7 @@ def search_page( **kwargs: Any, ) -> list[TextResult] | None: """Search a single page (for compatibility). - + Args: query: Search query region: Region code @@ -217,7 +217,7 @@ def search_page( timelimit: Time filter page: Page number **kwargs: Additional parameters - + Returns: List of results from the specified page """ @@ -229,14 +229,14 @@ def search_page( page=page, **kwargs ) - + html_text = self.request(self.search_method, self.search_url, params=payload) if not html_text: return None - + html_text = self.pre_process_html(html_text) results = self.extract_results(html_text) - + return self.post_extract_results(results) if results else None def run( @@ -249,7 +249,7 @@ def run( max_results: int | None = None, ) -> list[dict[str, str]]: """Run text search and return results as dictionaries. - + Args: keywords: Search query. region: Region code. @@ -257,7 +257,7 @@ def run( timelimit: Time filter. backend: Backend type (ignored for Yahoo). max_results: Maximum number of results. - + Returns: List of search result dictionaries. """ diff --git a/webscout/search/engines/yahoo/translate.py b/webscout/search/engines/yahoo/translate.py index 6e075383..973456f9 100644 --- a/webscout/search/engines/yahoo/translate.py +++ b/webscout/search/engines/yahoo/translate.py @@ -13,4 +13,4 @@ def run(self, *args, **kwargs) -> list[dict[str, str]]: Not supported. """ - raise NotImplementedError("Yahoo does not support translation") \ No newline at end of file + raise NotImplementedError("Yahoo does not support translation") diff --git a/webscout/search/engines/yahoo/videos.py b/webscout/search/engines/yahoo/videos.py index 5deb46bf..6c4378ae 100644 --- a/webscout/search/engines/yahoo/videos.py +++ b/webscout/search/engines/yahoo/videos.py @@ -6,13 +6,13 @@ from typing import Any from urllib.parse import parse_qs, urlparse -from .base import YahooSearchEngine from ...results import VideosResult +from .base import YahooSearchEngine class YahooVideos(YahooSearchEngine[VideosResult]): """Yahoo video search engine with filters. - + Features: - Length filters (short, medium, long) - Resolution filters (SD, HD, 4K) @@ -72,7 +72,7 @@ def build_payload( **kwargs: Any, ) -> dict[str, Any]: """Build video search payload. - + Args: query: Search query region: Region code @@ -83,7 +83,7 @@ def build_payload( - length: Video length filter - resolution: Video resolution filter - source: Video source filter - + Returns: Query parameters dictionary """ @@ -138,20 +138,20 @@ def build_payload( def extract_video_url(self, href: str) -> str: """Extract actual video URL from Yahoo redirect. - + Args: href: Yahoo redirect URL - + Returns: Actual video URL """ if not href: return href - + try: # Parse the URL parsed = urlparse(href) - + # Check if it's a Yahoo redirect if "r.search.yahoo.com" in parsed.netloc or "/RU=" in href: # Extract the RU parameter @@ -166,37 +166,37 @@ def extract_video_url(self, href: str) -> str: query_params = parse_qs(parsed.query) if "url" in query_params: return query_params["url"][0] - + return href except Exception: return href def post_extract_results(self, results: list[VideosResult]) -> list[VideosResult]: """Post-process video results. - + Args: results: Raw extracted results - + Returns: Cleaned results """ cleaned_results = [] - + for result in results: # Extract real URL if result.url: result.url = self.extract_video_url(result.url) - + # Skip invalid results if not result.url or not result.title: continue - + # Clean thumbnail URL if result.thumbnail and result.thumbnail.startswith("data:"): result.thumbnail = "" - + cleaned_results.append(result) - + return cleaned_results def search( @@ -210,7 +210,7 @@ def search( **kwargs: Any, ) -> list[VideosResult] | None: """Search Yahoo Videos with pagination. - + Args: query: Video search query region: Region code @@ -219,14 +219,14 @@ def search( page: Starting page max_results: Maximum results **kwargs: Additional filters (length, resolution, source) - + Returns: List of VideoResult objects """ results = [] current_page = page max_pages = kwargs.get("max_pages", 5) - + while current_page <= max_pages: payload = self.build_payload( query=query, @@ -236,29 +236,29 @@ def search( page=current_page, **kwargs ) - + html_text = self.request(self.search_method, self.search_url, params=payload) if not html_text: break - + html_text = self.pre_process_html(html_text) page_results = self.extract_results(html_text) - + if not page_results: break - + results.extend(page_results) - + if max_results and len(results) >= max_results: break - + current_page += 1 - + results = self.post_extract_results(results) - + if max_results: results = results[:max_results] - + return results if results else None def run( @@ -273,7 +273,7 @@ def run( max_results: int | None = None, ) -> list[dict[str, str]]: """Run video search and return results as dictionaries. - + Args: keywords: Search query. region: Region code. @@ -283,7 +283,7 @@ def run( duration: Video duration filter. license_videos: License filter. max_results: Maximum number of results. - + Returns: List of video result dictionaries. """ diff --git a/webscout/search/engines/yahoo/weather.py b/webscout/search/engines/yahoo/weather.py index 9044b9e4..c9ae4dcb 100644 --- a/webscout/search/engines/yahoo/weather.py +++ b/webscout/search/engines/yahoo/weather.py @@ -3,7 +3,6 @@ from __future__ import annotations import re -import json from typing import Any from ...http_client import HttpClient @@ -14,7 +13,7 @@ class YahooWeather: def __init__(self, proxy: str | None = None, timeout: int | None = None, verify: bool = True): """Initialize weather search engine. - + Args: proxy: Proxy URL. timeout: Request timeout in seconds. @@ -32,22 +31,22 @@ def request(self, method: str, url: str, **kwargs: Any) -> str | None: def run(self, *args, **kwargs) -> list[dict[str, Any]]: """Get weather data from Yahoo. - + Args: location: Location to get weather for (e.g., "New York", "London", "Bengaluru") - + Returns: List of weather data dictionaries """ location = args[0] if args else kwargs.get("location") or kwargs.get("keywords") - + if not location: raise ValueError("Location is required for weather search") - + try: # Use the search endpoint which redirects to the correct weather page search_url = f"https://weather.yahoo.com/search/?q={location.replace(' ', '+')}" - + # Fetch the page response = self.request("GET", search_url) if not response: @@ -55,25 +54,25 @@ def run(self, *args, **kwargs) -> list[dict[str, Any]]: "location": location, "error": "Failed to fetch weather data from Yahoo" }] - + # Extract JSON data from the page weather_data = self._extract_json_data(response, location) - + if weather_data: return [weather_data] - + # Fallback: try regex parsing return self._parse_weather_html(response, location) - + except Exception as e: return [{ "location": location, "error": f"Failed to fetch weather data: {str(e)}" }] - + def _extract_json_data(self, html: str, location: str) -> dict[str, Any] | None: """Extract weather data from embedded JSON in the page. - + Yahoo Weather embeds JSON data in script tags that can be parsed. """ try: @@ -81,53 +80,53 @@ def _extract_json_data(self, html: str, location: str) -> dict[str, Any] | None: # Pattern: self.__next_f.push([1,"..JSON data.."]) json_pattern = r'self\.__next_f\.push\(\[1,"([^"]+)"\]\)' matches = re.findall(json_pattern, html) - + weather_info = {} - + for match in matches: # Unescape the JSON string try: # The data is escaped, so we need to decode it decoded = match.encode().decode('unicode_escape') - + # Look for temperature data temp_match = re.search(r'"temperature":(\d+)', decoded) if temp_match and not weather_info.get('temperature'): weather_info['temperature'] = int(temp_match.group(1)) - + # Look for condition condition_match = re.search(r'"iconLabel":"([^"]+)"', decoded) if condition_match and not weather_info.get('condition'): weather_info['condition'] = condition_match.group(1) - + # Look for high/low high_match = re.search(r'"highTemperature":(\d+)', decoded) if high_match and not weather_info.get('high'): weather_info['high'] = int(high_match.group(1)) - + low_match = re.search(r'"lowTemperature":(\d+)', decoded) if low_match and not weather_info.get('low'): weather_info['low'] = int(low_match.group(1)) - + # Look for humidity humidity_match = re.search(r'"value":"(\d+)%"[^}]*"category":"Humidity"', decoded) if humidity_match and not weather_info.get('humidity'): weather_info['humidity'] = int(humidity_match.group(1)) - + # Look for precipitation probability precip_match = re.search(r'"probabilityOfPrecipitation":"(\d+)%"', decoded) if precip_match and not weather_info.get('precipitation_chance'): weather_info['precipitation_chance'] = int(precip_match.group(1)) - + # Look for location name location_match = re.search(r'"name":"([^"]+)","code":null,"woeid":(\d+)', decoded) if location_match and not weather_info.get('location_name'): weather_info['location_name'] = location_match.group(1) weather_info['woeid'] = int(location_match.group(2)) - + except Exception: continue - + if weather_info and weather_info.get('temperature'): return { "location": weather_info.get('location_name', location), @@ -141,78 +140,78 @@ def _extract_json_data(self, html: str, location: str) -> dict[str, Any] | None: "source": "Yahoo Weather", "units": "Fahrenheit" } - + return None - - except Exception as e: + + except Exception: return None - + def _parse_weather_html(self, html_content: str, location: str) -> list[dict[str, Any]]: """Fallback: Parse weather data from HTML content using regex. - + Args: html_content: HTML content of weather page location: Location name - + Returns: List of weather data dictionaries """ try: weather_data = {"location": location} - + # Extract current temperature temp_patterns = [ r']*class="[^"]*font-title1[^"]*"[^>]*>(\d+)°

', r'>(\d+)°<', r'"temperature":(\d+)', ] - + for pattern in temp_patterns: match = re.search(pattern, html_content) if match: weather_data["temperature_f"] = int(match.group(1)) break - + # Extract condition condition_patterns = [ r'"iconLabel":"([^"]+)"', r'aria-label="([^"]*(?:Cloudy|Sunny|Rain|Clear|Thunder|Shower|Fog)[^"]*)"', ] - + for pattern in condition_patterns: match = re.search(pattern, html_content, re.IGNORECASE) if match: weather_data["condition"] = match.group(1) break - + # Extract high/low high_match = re.search(r'"highTemperature":(\d+)', html_content) if high_match: weather_data["high_f"] = int(high_match.group(1)) - + low_match = re.search(r'"lowTemperature":(\d+)', html_content) if low_match: weather_data["low_f"] = int(low_match.group(1)) - + # Extract humidity humidity_match = re.search(r'Humidity[^>]*>(\d+)%|"value":"(\d+)%"[^}]*"Humidity"', html_content, re.IGNORECASE) if humidity_match: weather_data["humidity_percent"] = int(humidity_match.group(1) or humidity_match.group(2)) - + weather_data["source"] = "Yahoo Weather" weather_data["units"] = "Fahrenheit" - + # Remove None values weather_data = {k: v for k, v in weather_data.items() if v is not None} - + if len(weather_data) > 3: # Has more than just location, source, and units return [weather_data] - + return [{ "location": location, "error": "Could not extract weather data from page" }] - + except Exception as e: return [{ "location": location, diff --git a/webscout/search/engines/yandex.py b/webscout/search/engines/yandex.py index 59b095d9..d79300c6 100644 --- a/webscout/search/engines/yandex.py +++ b/webscout/search/engines/yandex.py @@ -44,13 +44,13 @@ def build_payload( def run(self, *args, **kwargs) -> list[TextResult]: """Run text search on Yandex. - + Args: keywords: Search query. region: Region code. safesearch: Safe search level. max_results: Maximum number of results (ignored for now). - + Returns: List of TextResult objects. """ @@ -58,7 +58,7 @@ def run(self, *args, **kwargs) -> list[TextResult]: region = args[1] if len(args) > 1 else kwargs.get("region", "us-en") safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate") max_results = args[3] if len(args) > 3 else kwargs.get("max_results") - + results = self.search(query=keywords, region=region, safesearch=safesearch) if results and max_results: results = results[:max_results] diff --git a/webscout/search/engines/yep/base.py b/webscout/search/engines/yep/base.py index ff3dac76..297d9ccc 100644 --- a/webscout/search/engines/yep/base.py +++ b/webscout/search/engines/yep/base.py @@ -1,8 +1,9 @@ from __future__ import annotations -from ....litagent import LitAgent from curl_cffi.requests import Session +from ....litagent import LitAgent + class YepBase: """Base class for Yep search engines.""" diff --git a/webscout/search/engines/yep/images.py b/webscout/search/engines/yep/images.py index aa6fedfc..131762f8 100644 --- a/webscout/search/engines/yep/images.py +++ b/webscout/search/engines/yep/images.py @@ -1,11 +1,12 @@ from __future__ import annotations -from typing import List, Optional +from typing import List from urllib.parse import urlencode -from .base import YepBase from webscout.search.results import ImagesResult +from .base import YepBase + class YepImages(YepBase): name = "yep" diff --git a/webscout/search/engines/yep/text.py b/webscout/search/engines/yep/text.py index 6b97d794..a739ec12 100644 --- a/webscout/search/engines/yep/text.py +++ b/webscout/search/engines/yep/text.py @@ -1,11 +1,12 @@ from __future__ import annotations -from typing import List, Optional +from typing import List from urllib.parse import urlencode -from .base import YepBase from webscout.search.results import TextResult +from .base import YepBase + class YepSearch(YepBase): name = "yep" diff --git a/webscout/search/http_client.py b/webscout/search/http_client.py index 8f68631f..fc8f4da0 100644 --- a/webscout/search/http_client.py +++ b/webscout/search/http_client.py @@ -2,7 +2,6 @@ from __future__ import annotations -from litprinter import ic from random import choice from typing import Any, Literal @@ -18,7 +17,7 @@ class HttpClient: """HTTP client wrapper for search engines.""" - + # curl_cffi supported browser impersonations _impersonates = ( "chrome99", "chrome100", "chrome101", "chrome104", "chrome107", "chrome110", @@ -37,7 +36,7 @@ def __init__( headers: dict[str, str] | None = None, ) -> None: """Initialize HTTP client. - + Args: proxy: Proxy URL (supports http/https/socks5). timeout: Request timeout in seconds. @@ -47,10 +46,10 @@ def __init__( self.proxy = proxy self.timeout = timeout self.verify = verify - + # Choose random browser to impersonate impersonate_browser = choice(self._impersonates) - + # Initialize curl_cffi session self.client = curl_cffi.requests.Session( headers=headers or {}, @@ -59,7 +58,7 @@ def __init__( impersonate=impersonate_browser, verify=verify, ) - + def request( self, method: Literal["GET", "POST", "HEAD", "OPTIONS", "DELETE", "PUT", "PATCH"], @@ -73,7 +72,7 @@ def request( **kwargs: Any, ) -> curl_cffi.requests.Response: """Make HTTP request. - + Args: method: HTTP method. url: Request URL. @@ -84,10 +83,10 @@ def request( cookies: Request cookies. timeout: Request timeout (overrides default). **kwargs: Additional arguments passed to curl_cffi. - + Returns: Response object. - + Raises: TimeoutE: Request timeout. RatelimitE: Rate limit exceeded. @@ -101,15 +100,15 @@ def request( "timeout": timeout or self.timeout, **kwargs, } - + if isinstance(cookies, dict): request_kwargs["cookies"] = cookies - + if data is not None: request_kwargs["data"] = data - + resp = self.client.request(method, url, **request_kwargs) - + # Check response status if resp.status_code == 200: return resp @@ -117,38 +116,38 @@ def request( raise RatelimitE(f"{resp.url} {resp.status_code} Rate limit") else: raise WebscoutE(f"{resp.url} returned {resp.status_code}") - + except Exception as ex: if "time" in str(ex).lower() or "timeout" in str(ex).lower(): raise TimeoutE(f"{url} {type(ex).__name__}: {ex}") from ex raise WebscoutE(f"{url} {type(ex).__name__}: {ex}") from ex - + def get(self, url: str, **kwargs: Any) -> curl_cffi.requests.Response: """Make GET request.""" return self.request("GET", url, **kwargs) - + def post(self, url: str, **kwargs: Any) -> curl_cffi.requests.Response: """Make POST request.""" return self.request("POST", url, **kwargs) - + def set_cookies(self, url: str, cookies: dict[str, str]) -> None: """Set cookies for a domain. - + Args: url: URL to set cookies for. cookies: Cookie dictionary. """ self.client.cookies.update(cookies) - + def close(self) -> None: """Close the HTTP client.""" if hasattr(self.client, 'close'): self.client.close() - + def __enter__(self) -> HttpClient: """Context manager entry.""" return self - + def __exit__(self, *args: Any) -> None: """Context manager exit.""" self.close() diff --git a/webscout/search/results.py b/webscout/search/results.py index e9d0cc7e..a740f603 100644 --- a/webscout/search/results.py +++ b/webscout/search/results.py @@ -9,11 +9,11 @@ @dataclass class TextResult: """Text search result.""" - + title: str = "" href: str = "" body: str = "" - + def to_dict(self) -> dict[str, Any]: """Convert to dictionary.""" return { @@ -26,7 +26,7 @@ def to_dict(self) -> dict[str, Any]: @dataclass class ImagesResult: """Images search result.""" - + title: str = "" image: str = "" thumbnail: str = "" @@ -34,7 +34,7 @@ class ImagesResult: height: int = 0 width: int = 0 source: str = "" - + def to_dict(self) -> dict[str, Any]: """Convert to dictionary.""" return { @@ -51,7 +51,7 @@ def to_dict(self) -> dict[str, Any]: @dataclass class VideosResult: """Videos search result.""" - + content: str = "" description: str = "" duration: str = "" @@ -65,7 +65,7 @@ class VideosResult: statistics: dict[str, int] = field(default_factory=dict) title: str = "" uploader: str = "" - + def to_dict(self) -> dict[str, Any]: """Convert to dictionary.""" return { @@ -88,14 +88,14 @@ def to_dict(self) -> dict[str, Any]: @dataclass class NewsResult: """News search result.""" - + date: str = "" title: str = "" body: str = "" url: str = "" image: str = "" source: str = "" - + def to_dict(self) -> dict[str, Any]: """Convert to dictionary.""" return { @@ -111,7 +111,7 @@ def to_dict(self) -> dict[str, Any]: @dataclass class BooksResult: """Books search result.""" - + title: str = "" author: str = "" href: str = "" @@ -121,7 +121,7 @@ class BooksResult: language: str = "" filesize: str = "" extension: str = "" - + def to_dict(self) -> dict[str, Any]: """Convert to dictionary.""" return { diff --git a/webscout/search/yahoo_main.py b/webscout/search/yahoo_main.py index d7f1bdf1..777a7e28 100644 --- a/webscout/search/yahoo_main.py +++ b/webscout/search/yahoo_main.py @@ -1,19 +1,21 @@ """Yahoo unified search interface.""" from __future__ import annotations -from typing import List, Optional + +from typing import Dict, List, Optional + from .base import BaseSearch -from .engines.yahoo.text import YahooText +from .engines.yahoo.answers import YahooAnswers from .engines.yahoo.images import YahooImages -from .engines.yahoo.videos import YahooVideos +from .engines.yahoo.maps import YahooMaps from .engines.yahoo.news import YahooNews from .engines.yahoo.suggestions import YahooSuggestions -from .engines.yahoo.answers import YahooAnswers -from .engines.yahoo.maps import YahooMaps +from .engines.yahoo.text import YahooText from .engines.yahoo.translate import YahooTranslate +from .engines.yahoo.videos import YahooVideos from .engines.yahoo.weather import YahooWeather -from .results import TextResult, ImagesResult, VideosResult, NewsResult -from typing import Dict +from .results import ImagesResult, NewsResult, TextResult, VideosResult + class YahooSearch(BaseSearch): """Unified Yahoo search interface.""" @@ -52,4 +54,4 @@ def translate(self, keywords: str, from_lang: Optional[str] = None, to_lang: str def weather(self, keywords: str) -> List[Dict[str, str]]: search = YahooWeather() - return search.run(keywords) \ No newline at end of file + return search.run(keywords) diff --git a/webscout/search/yep_main.py b/webscout/search/yep_main.py index 7ce1837c..e8234cd1 100644 --- a/webscout/search/yep_main.py +++ b/webscout/search/yep_main.py @@ -1,11 +1,13 @@ """Yep unified search interface.""" from __future__ import annotations + from typing import Dict, List, Optional + from .base import BaseSearch -from .engines.yep.text import YepSearch as YepTextSearch from .engines.yep.images import YepImages from .engines.yep.suggestions import YepSuggestions +from .engines.yep.text import YepSearch as YepTextSearch class YepSearch(BaseSearch): diff --git a/webscout/server/__init__.py b/webscout/server/__init__.py index 51fa9f14..bc912bd7 100644 --- a/webscout/server/__init__.py +++ b/webscout/server/__init__.py @@ -1,4 +1,7 @@ # webscout/server/__init__.py +from .exceptions import APIError +from .routes import Api + # Import server functions lazily to avoid module execution issues def create_app(): @@ -12,8 +15,8 @@ def run_api(*args, **kwargs): def start_server(*args, **kwargs): from .server import start_server as _start_server return _start_server(*args, **kwargs) -from .routes import Api -from .exceptions import APIError + + # Lazy imports for config classes to avoid initialization issues def get_server_config(): @@ -42,4 +45,4 @@ def initialize_tti_provider_map(): "APIError", "initialize_provider_map", "initialize_tti_provider_map" -] \ No newline at end of file +] diff --git a/webscout/server/config.py b/webscout/server/config.py index 5e0faaa2..61f37f62 100644 --- a/webscout/server/config.py +++ b/webscout/server/config.py @@ -3,7 +3,8 @@ """ import os -from typing import List, Dict, Optional, Any +from typing import Any, Dict, List, Optional + from litprinter import ic # Configuration constants @@ -33,7 +34,8 @@ def update(self, **kwargs) -> None: for key, value in kwargs.items(): if hasattr(self, key) and value is not None: setattr(self, key, value) - ic.configureOutput(prefix='INFO| '); ic(f"Config updated: {key} = {value}") + ic.configureOutput(prefix='INFO| ') + ic(f"Config updated: {key} = {value}") def validate(self) -> None: """Validate configuration settings.""" @@ -42,7 +44,8 @@ def validate(self) -> None: if self.default_provider not in self.provider_map and self.provider_map: available_providers = list(set(v.__name__ for v in self.provider_map.values())) - ic.configureOutput(prefix='WARNING| '); ic(f"Default provider '{self.default_provider}' not found. Available: {available_providers}") + ic.configureOutput(prefix='WARNING| ') + ic(f"Default provider '{self.default_provider}' not found. Available: {available_providers}") class AppConfig: @@ -72,4 +75,4 @@ def set_config(cls, **data): config.update(**filtered_data) except ImportError: # Handle case where server module is not available - pass \ No newline at end of file + pass diff --git a/webscout/server/exceptions.py b/webscout/server/exceptions.py index be3bb106..00b2cb8e 100644 --- a/webscout/server/exceptions.py +++ b/webscout/server/exceptions.py @@ -5,8 +5,10 @@ import json import re from typing import Optional + from fastapi.responses import JSONResponse from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR + from .request_models import ErrorDetail, ErrorResponse @@ -14,10 +16,10 @@ def clean_text(text): """Clean text by removing null bytes and control characters except newlines and tabs.""" if not isinstance(text, str): return text - + # Remove null bytes text = text.replace('\x00', '') - + # Keep newlines, tabs, and other printable characters, remove other control chars # This regex matches control characters except \n, \r, \t return re.sub(r'[\x01-\x08\x0b\x0c\x0e-\x1f\x7f]', '', text) diff --git a/webscout/server/providers.py b/webscout/server/providers.py index 45dee819..c9f1478e 100644 --- a/webscout/server/providers.py +++ b/webscout/server/providers.py @@ -2,11 +2,12 @@ Provider management and initialization for the Webscout API. """ -import sys import inspect +import sys from typing import Any, Dict, Tuple -from starlette.status import HTTP_404_NOT_FOUND, HTTP_500_INTERNAL_SERVER_ERROR + from litprinter import ic +from starlette.status import HTTP_404_NOT_FOUND, HTTP_500_INTERNAL_SERVER_ERROR from .config import AppConfig from .exceptions import APIError @@ -18,7 +19,8 @@ def initialize_provider_map() -> None: """Initialize the provider map by discovering available providers.""" - ic.configureOutput(prefix='INFO| '); ic("Initializing provider map...") + ic.configureOutput(prefix='INFO| ') + ic("Initializing provider map...") try: from webscout.Provider.OPENAI.base import OpenAICompatibleProvider @@ -34,7 +36,7 @@ def initialize_provider_map() -> None: and obj.__name__ != "OpenAICompatibleProvider" ): # Only include providers that don't require authentication - if hasattr(obj, 'required_auth') and getattr(obj, 'required_auth', True) == False: + if hasattr(obj, 'required_auth') and not getattr(obj, 'required_auth', True): provider_name = obj.__name__ AppConfig.provider_map[provider_name] = obj provider_count += 1 @@ -51,7 +53,8 @@ def initialize_provider_map() -> None: # Fallback to ChatGPT if no providers found if not AppConfig.provider_map: - ic.configureOutput(prefix='WARNING| '); ic("No providers found, using ChatGPT fallback") + ic.configureOutput(prefix='WARNING| ') + ic("No providers found, using ChatGPT fallback") try: from webscout.Provider.OPENAI.chatgpt import ChatGPT fallback_models = ["gpt-4", "gpt-4o", "gpt-4o-mini", "gpt-3.5-turbo"] @@ -66,19 +69,23 @@ def initialize_provider_map() -> None: provider_count = 1 model_count = len(fallback_models) except ImportError as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Failed to import ChatGPT fallback: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Failed to import ChatGPT fallback: {e}") raise APIError("No providers available", HTTP_500_INTERNAL_SERVER_ERROR) - ic.configureOutput(prefix='INFO| '); ic(f"Initialized {provider_count} providers with {model_count} models") + ic.configureOutput(prefix='INFO| ') + ic(f"Initialized {provider_count} providers with {model_count} models") except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Failed to initialize provider map: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Failed to initialize provider map: {e}") raise APIError(f"Provider initialization failed: {e}", HTTP_500_INTERNAL_SERVER_ERROR) def initialize_tti_provider_map() -> None: """Initialize the TTI provider map by discovering available TTI providers.""" - ic.configureOutput(prefix='INFO| '); ic("Initializing TTI provider map...") + ic.configureOutput(prefix='INFO| ') + ic("Initializing TTI provider map...") try: from webscout.Provider.TTI.base import TTICompatibleProvider @@ -110,7 +117,8 @@ def initialize_tti_provider_map() -> None: # Fallback to PollinationsAI if no TTI providers found if not AppConfig.tti_provider_map: - ic.configureOutput(prefix='WARNING| '); ic("No TTI providers found, using PollinationsAI fallback") + ic.configureOutput(prefix='WARNING| ') + ic("No TTI providers found, using PollinationsAI fallback") try: from webscout.Provider.TTI.pollinations import PollinationsAI fallback_models = ["flux", "turbo", "gptimage"] @@ -125,13 +133,16 @@ def initialize_tti_provider_map() -> None: provider_count = 1 model_count = len(fallback_models) except ImportError as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Failed to import PollinationsAI fallback: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Failed to import PollinationsAI fallback: {e}") raise APIError("No TTI providers available", HTTP_500_INTERNAL_SERVER_ERROR) - ic.configureOutput(prefix='INFO| '); ic(f"Initialized {provider_count} TTI providers with {model_count} models") + ic.configureOutput(prefix='INFO| ') + ic(f"Initialized {provider_count} TTI providers with {model_count} models") except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Failed to initialize TTI provider map: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Failed to initialize TTI provider map: {e}") raise APIError(f"TTI Provider initialization failed: {e}", HTTP_500_INTERNAL_SERVER_ERROR) diff --git a/webscout/server/request_models.py b/webscout/server/request_models.py index 2bdd8fa2..284f374d 100644 --- a/webscout/server/request_models.py +++ b/webscout/server/request_models.py @@ -2,7 +2,8 @@ Pydantic models for API requests and responses. """ -from typing import List, Dict, Optional, Union, Literal +from typing import Dict, List, Literal, Optional, Union + from webscout.Provider.OPENAI.pydantic_imports import BaseModel, Field diff --git a/webscout/server/request_processing.py b/webscout/server/request_processing.py index 9017ffca..987b73b9 100644 --- a/webscout/server/request_processing.py +++ b/webscout/server/request_processing.py @@ -5,18 +5,23 @@ import json import time import uuid -from typing import List, Dict, Any -from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY, HTTP_500_INTERNAL_SERVER_ERROR -from fastapi.responses import StreamingResponse +from typing import Any, Dict, List -from webscout.Provider.OPENAI.utils import ChatCompletion, Choice, ChatCompletionMessage, CompletionUsage +from fastapi.responses import StreamingResponse from litprinter import ic +from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY, HTTP_500_INTERNAL_SERVER_ERROR -from .request_models import Message, ChatCompletionRequest -from .exceptions import APIError, clean_text +from webscout.Provider.OPENAI.utils import ( + ChatCompletion, + ChatCompletionMessage, + Choice, + CompletionUsage, +) # from .simple_logger import log_api_request, get_client_ip, generate_request_id from .config import AppConfig +from .exceptions import APIError, clean_text +from .request_models import ChatCompletionRequest, Message def get_client_ip(request) -> str: @@ -24,11 +29,11 @@ def get_client_ip(request) -> str: forwarded_for = request.headers.get("X-Forwarded-For") if forwarded_for: return forwarded_for.split(",")[0].strip() - + real_ip = request.headers.get("X-Real-IP") if real_ip: return real_ip.strip() - + return getattr(request.client, "host", "unknown") @@ -60,7 +65,7 @@ async def log_request(request_id: str, ip_address: str, model_used: str, questio user_agent = None if request_obj: user_agent = request_obj.headers.get("user-agent") - + await log_api_request( request_id=request_id, ip_address=ip_address, @@ -73,7 +78,8 @@ async def log_request(request_id: str, ip_address: str, model_used: str, questio user_agent=user_agent ) except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Failed to log request {request_id}: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Failed to log request {request_id}: {e}") # Don't raise exception to avoid breaking the main request flow @@ -134,15 +140,16 @@ def prepare_provider_params(chat_request: ChatCompletionRequest, model_name: str async def handle_streaming_response(provider: Any, params: Dict[str, Any], request_id: str, - ip_address: str, question: str, model_name: str, start_time: float, + ip_address: str, question: str, model_name: str, start_time: float, provider_name: str = None, request_obj=None) -> StreamingResponse: """Handle streaming chat completion response.""" collected_content = [] - + async def streaming(): nonlocal collected_content try: - ic.configureOutput(prefix='DEBUG| '); ic(f"Starting streaming response for request {request_id}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Starting streaming response for request {request_id}") completion_stream = provider.chat.completions.create(**params) # Check if it's iterable (generator, iterator, or other iterable types) @@ -158,7 +165,7 @@ async def streaming(): chunk_data = chunk else: # Fallback for unknown chunk types chunk_data = chunk - + # Clean text content in the chunk to remove control characters if isinstance(chunk_data, dict) and 'choices' in chunk_data: for choice in chunk_data.get('choices', []): @@ -175,10 +182,11 @@ async def streaming(): if content: collected_content.append(content) choice['message']['content'] = clean_text(content) - + yield f"data: {json.dumps(chunk_data, ensure_ascii=False)}\n\n" except TypeError as te: - ic.configureOutput(prefix='ERROR| '); ic(f"Error iterating over completion_stream: {te}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Error iterating over completion_stream: {te}") # Fall back to treating as non-generator response if hasattr(completion_stream, 'model_dump'): response_data = completion_stream.model_dump(exclude_none=True) @@ -186,7 +194,7 @@ async def streaming(): response_data = completion_stream.dict(exclude_none=True) else: response_data = completion_stream - + # Clean text content in the response if isinstance(response_data, dict) and 'choices' in response_data: for choice in response_data.get('choices', []): @@ -201,7 +209,7 @@ async def streaming(): if content: collected_content.append(content) choice['message']['content'] = clean_text(content) - + yield f"data: {json.dumps(response_data, ensure_ascii=False)}\n\n" else: # Non-generator response if hasattr(completion_stream, 'model_dump'): @@ -210,7 +218,7 @@ async def streaming(): response_data = completion_stream.dict(exclude_none=True) else: response_data = completion_stream - + # Clean text content in the response if isinstance(response_data, dict) and 'choices' in response_data: for choice in response_data.get('choices', []): @@ -225,11 +233,12 @@ async def streaming(): if content: collected_content.append(content) choice['message']['content'] = clean_text(content) - + yield f"data: {json.dumps(response_data, ensure_ascii=False)}\n\n" except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Error in streaming response for request {request_id}: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Error in streaming response for request {request_id}: {e}") error_message = clean_text(str(e)) error_data = { "error": { @@ -239,7 +248,7 @@ async def streaming(): } } yield f"data: {json.dumps(error_data, ensure_ascii=False)}\n\n" - + # Log error request response_time_ms = int((time.time() - start_time) * 1000) await log_request( @@ -256,7 +265,7 @@ async def streaming(): ) finally: yield "data: [DONE]\n\n" - + # Log successful streaming request if collected_content: answer = "".join(collected_content) @@ -272,17 +281,18 @@ async def streaming(): provider=provider_name, request_obj=request_obj ) - + return StreamingResponse(streaming(), media_type="text/event-stream") async def handle_non_streaming_response(provider: Any, params: Dict[str, Any], request_id: str, start_time: float, ip_address: str, - question: str, model_name: str, provider_name: str = None, + question: str, model_name: str, provider_name: str = None, request_obj=None) -> Dict[str, Any]: """Handle non-streaming chat completion response.""" try: - ic.configureOutput(prefix='DEBUG| '); ic(f"Starting non-streaming response for request {request_id}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Starting non-streaming response for request {request_id}") completion = provider.chat.completions.create(**params) if completion is None: @@ -298,7 +308,7 @@ async def handle_non_streaming_response(provider: Any, params: Dict[str, Any], )], usage=CompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0) ).model_dump(exclude_none=True) - + # Log error request response_time_ms = int((time.time() - start_time) * 1000) await log_request( @@ -313,7 +323,7 @@ async def handle_non_streaming_response(provider: Any, params: Dict[str, Any], provider=provider_name, request_obj=request_obj ) - + return error_response # Standardize response format @@ -329,7 +339,7 @@ async def handle_non_streaming_response(provider: Any, params: Dict[str, Any], HTTP_500_INTERNAL_SERVER_ERROR, "provider_error" ) - + # Extract answer from response and clean text content answer = "" if isinstance(response_data, dict) and 'choices' in response_data: @@ -343,7 +353,8 @@ async def handle_non_streaming_response(provider: Any, params: Dict[str, Any], elapsed = time.time() - start_time response_time_ms = int(elapsed * 1000) - ic.configureOutput(prefix='INFO| '); ic(f"Completed non-streaming request {request_id} in {elapsed:.2f}s") + ic.configureOutput(prefix='INFO| ') + ic(f"Completed non-streaming request {request_id} in {elapsed:.2f}s") # Log successful request await log_request( @@ -361,9 +372,10 @@ async def handle_non_streaming_response(provider: Any, params: Dict[str, Any], return response_data except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Error in non-streaming response for request {request_id}: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Error in non-streaming response for request {request_id}: {e}") error_message = clean_text(str(e)) - + # Log error request response_time_ms = int((time.time() - start_time) * 1000) await log_request( @@ -378,7 +390,7 @@ async def handle_non_streaming_response(provider: Any, params: Dict[str, Any], provider=provider_name, request_obj=request_obj ) - + raise APIError( f"Provider error: {error_message}", HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/webscout/server/routes.py b/webscout/server/routes.py index 0870d9cf..2a0a9dc6 100644 --- a/webscout/server/routes.py +++ b/webscout/server/routes.py @@ -5,49 +5,52 @@ import time import uuid -from fastapi import FastAPI, Request, Body, Query -from fastapi.responses import JSONResponse +from fastapi import Body, FastAPI, Query, Request from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from litprinter import ic from starlette.exceptions import HTTPException as StarletteHTTPException from starlette.status import ( HTTP_500_INTERNAL_SERVER_ERROR, ) -from litprinter import ic +from webscout.search.engines import ENGINES from .config import AppConfig -from .request_models import ( - ChatCompletionRequest, ImageGenerationRequest, ModelListResponse -) from .exceptions import APIError from .providers import ( - resolve_provider_and_model, resolve_tti_provider_and_model, - get_provider_instance, get_tti_provider_instance + get_provider_instance, + get_tti_provider_instance, + resolve_provider_and_model, + resolve_tti_provider_and_model, ) +from .request_models import ChatCompletionRequest, ImageGenerationRequest, ModelListResponse from .request_processing import ( - process_messages, prepare_provider_params, - handle_streaming_response, handle_non_streaming_response + handle_non_streaming_response, + handle_streaming_response, + prepare_provider_params, + process_messages, ) -from webscout.search.engines import ENGINES + class Api: """API route handler class.""" - + def __init__(self, app: FastAPI) -> None: self.app = app def register_validation_exception_handler(self): """Register comprehensive exception handlers.""" - from fastapi.exceptions import RequestValidationError - from starlette.exceptions import HTTPException as StarletteHTTPException from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY, HTTP_500_INTERNAL_SERVER_ERROR + from .exceptions import APIError - + github_footer = "If you believe this is a bug, please pull an issue at https://github.com/OEvortex/Webscout." @self.app.exception_handler(APIError) async def api_error_handler(request, exc: APIError): - ic.configureOutput(prefix='ERROR| '); ic(f"API Error: {exc.message} (Status: {exc.status_code})") + ic.configureOutput(prefix='ERROR| ') + ic(f"API Error: {exc.message} (Status: {exc.status_code})") # Patch: add footer to error content before creating JSONResponse error_response = exc.to_response() # If the response is a JSONResponse, patch its content dict before returning @@ -68,7 +71,7 @@ async def validation_exception_handler(request, exc: RequestValidationError): errors = exc.errors() error_messages = [] body = await request.body() - is_empty_body = not body or body.strip() in (b"", b"null", b"{}") + not body or body.strip() in (b"", b"null", b"{}") for error in errors: loc = error.get("loc", []) loc_str = " -> ".join(str(item) for item in loc) @@ -101,7 +104,8 @@ async def http_exception_handler(request, exc: StarletteHTTPException): @self.app.exception_handler(Exception) async def general_exception_handler(request, exc: Exception): - ic.configureOutput(prefix='ERROR| '); ic(f"Unhandled server error: {exc}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Unhandled server error: {exc}") content = { "error": { "message": f"Internal server error: {str(exc)}", @@ -126,7 +130,7 @@ async def health_check(): return {"status": "healthy", "service": "webscout-api", "version": "0.2.0"} def _register_model_routes(self): - """Register model listing routes.""" + """Register model listing routes.""" @self.app.get( "/v1/models", response_model=ModelListResponse, @@ -183,32 +187,32 @@ async def retrieve_model(model: str): async def list_providers(): """Get information about all available chat completion providers.""" providers = {} - + # Extract unique provider names (exclude model mappings) provider_names = set() for key, provider_class in AppConfig.provider_map.items(): if "/" not in key: # Provider name, not model mapping provider_names.add(key) - + for provider_name in sorted(provider_names): provider_class = AppConfig.provider_map[provider_name] - + # Get available models for this provider models = [] for key, cls in AppConfig.provider_map.items(): if key.startswith(f"{provider_name}/"): model_name = key.split("/", 1)[1] models.append(model_name) - + # Sort models models = sorted(models) - + # Get supported parameters (common OpenAI-compatible parameters) supported_params = [ - "model", "messages", "max_tokens", "temperature", "top_p", + "model", "messages", "max_tokens", "temperature", "top_p", "presence_penalty", "frequency_penalty", "stop", "stream", "user" ] - + providers[provider_name] = { "name": provider_name, "class": provider_class.__name__, @@ -216,12 +220,12 @@ async def list_providers(): "parameters": supported_params, "model_count": len(models) } - + return { "providers": providers, "total_providers": len(providers) } - + @self.app.get( "/v1/TTI/models", response_model=ModelListResponse, @@ -256,32 +260,32 @@ async def list_tti_models(): async def list_tti_providers(): """Get information about all available TTI providers.""" providers = {} - + # Extract unique provider names (exclude model mappings) provider_names = set() for key, provider_class in AppConfig.tti_provider_map.items(): if "/" not in key: # Provider name, not model mapping provider_names.add(key) - + for provider_name in sorted(provider_names): provider_class = AppConfig.tti_provider_map[provider_name] - + # Get available models for this provider models = [] for key, cls in AppConfig.tti_provider_map.items(): if key.startswith(f"{provider_name}/"): model_name = key.split("/", 1)[1] models.append(model_name) - + # Sort models models = sorted(models) - + # Get supported parameters (common TTI parameters) supported_params = [ - "prompt", "model", "n", "size", "response_format", "user", + "prompt", "model", "n", "size", "response_format", "user", "style", "aspect_ratio", "timeout", "image_format", "seed" ] - + providers[provider_name] = { "name": provider_name, "class": provider_class.__name__, @@ -289,7 +293,7 @@ async def list_tti_providers(): "parameters": supported_params, "model_count": len(models) } - + return { "providers": providers, "total_providers": len(providers) @@ -325,7 +329,8 @@ async def chat_completions( request_id = f"chatcmpl-{uuid.uuid4()}" try: - ic.configureOutput(prefix='INFO| '); ic(f"Processing chat completion request {request_id} for model: {chat_request.model}") + ic.configureOutput(prefix='INFO| ') + ic(f"Processing chat completion request {request_id} for model: {chat_request.model}") # Resolve provider and model provider_class, model_name = resolve_provider_and_model(chat_request.model) @@ -333,9 +338,11 @@ async def chat_completions( # Initialize provider with caching and error handling try: provider = get_provider_instance(provider_class) - ic.configureOutput(prefix='DEBUG| '); ic(f"Using provider instance: {provider_class.__name__}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Using provider instance: {provider_class.__name__}") except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Failed to initialize provider {provider_class.__name__}: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Failed to initialize provider {provider_class.__name__}: {e}") raise APIError( f"Failed to initialize provider {provider_class.__name__}: {e}", HTTP_500_INTERNAL_SERVER_ERROR, @@ -386,7 +393,8 @@ async def chat_completions( # Re-raise API errors as-is raise except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Unexpected error in chat completion {request_id}: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Unexpected error in chat completion {request_id}: {e}") raise APIError( f"Internal server error: {str(e)}", HTTP_500_INTERNAL_SERVER_ERROR, @@ -407,7 +415,8 @@ async def image_generations( request_id = f"img-{uuid.uuid4()}" try: - ic.configureOutput(prefix='INFO| '); ic(f"Processing image generation request {request_id} for model: {image_request.model}") + ic.configureOutput(prefix='INFO| ') + ic(f"Processing image generation request {request_id} for model: {image_request.model}") # Resolve TTI provider and model provider_class, model_name = resolve_tti_provider_and_model(image_request.model) @@ -415,7 +424,8 @@ async def image_generations( # Initialize TTI provider try: provider = get_tti_provider_instance(provider_class) - ic.configureOutput(prefix='DEBUG| '); ic(f"Using TTI provider instance: {provider_class.__name__}") + ic.configureOutput(prefix='DEBUG| ') + ic(f"Using TTI provider instance: {provider_class.__name__}") except APIError as e: # Add helpful footer for provider errors return JSONResponse( @@ -429,7 +439,8 @@ async def image_generations( } ) except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Failed to initialize TTI provider {provider_class.__name__}: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Failed to initialize TTI provider {provider_class.__name__}: {e}") raise APIError( f"Failed to initialize TTI provider {provider_class.__name__}: {e}", HTTP_500_INTERNAL_SERVER_ERROR, @@ -470,13 +481,15 @@ async def image_generations( ) elapsed = time.time() - start_time - ic.configureOutput(prefix='INFO| '); ic(f"Completed image generation request {request_id} in {elapsed:.2f}s") + ic.configureOutput(prefix='INFO| ') + ic(f"Completed image generation request {request_id} in {elapsed:.2f}s") return response_data except APIError: raise except Exception as e: - ic.configureOutput(prefix='ERROR| '); ic(f"Unexpected error in image generation {request_id}: {e}") + ic.configureOutput(prefix='ERROR| ') + ic(f"Unexpected error in image generation {request_id}: {e}") raise APIError( f"Internal server error: {str(e)}", HTTP_500_INTERNAL_SERVER_ERROR, @@ -576,47 +589,47 @@ async def websearch( async def get_search_providers(): """Get information about all available search providers.""" providers = {} - + # Collect all unique engine names all_engines = set() for category_engines in ENGINES.values(): all_engines.update(category_engines.keys()) - + for engine_name in sorted(all_engines): # Find all categories this engine supports categories = [] for category, engines in ENGINES.items(): if engine_name in engines: categories.append(category) - + # Get supported parameters based on categories supported_params = ["q"] # query is always supported - + if "text" in categories or "images" in categories or "news" in categories or "videos" in categories: supported_params.extend(["max_results", "region", "safesearch"]) - + if "suggestions" in categories: supported_params.extend(["region"]) - + if "maps" in categories: supported_params.extend(["place", "street", "city", "county", "state", "country", "postalcode", "latitude", "longitude", "radius", "max_results"]) - + if "translate" in categories: supported_params.extend(["from_", "to"]) - + if "weather" in categories: supported_params.extend(["language"]) - + # Remove duplicates supported_params = list(set(supported_params)) - + providers[engine_name] = { "name": engine_name, "categories": sorted(categories), "supported_types": sorted(categories), # types are the same as categories "parameters": sorted(supported_params) } - + return { "providers": providers, "total_providers": len(providers) diff --git a/webscout/server/server.py b/webscout/server/server.py index faf72c70..161a25f0 100644 --- a/webscout/server/server.py +++ b/webscout/server/server.py @@ -13,13 +13,12 @@ import uvicorn from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import RedirectResponse from fastapi.openapi.docs import get_swagger_ui_html from starlette.responses import HTMLResponse -from .config import ServerConfig, AppConfig -from .routes import Api +from .config import AppConfig, ServerConfig from .providers import initialize_provider_map, initialize_tti_provider_map +from .routes import Api from .ui_templates import LANDING_PAGE_HTML, SWAGGER_CSS # Configuration constants @@ -66,14 +65,14 @@ async def custom_swagger_ui_html(): openapi_url=app.openapi_url, title=app.title + " - API Documentation", ).body.decode("utf-8") - + # Custom footer and styles footer_html = """ """ - + # Inject custom CSS and footer html = html.replace("", f"") html = html.replace("", f"{footer_html}") @@ -99,7 +98,7 @@ async def startup(): api = Api(app) api.register_validation_exception_handler() api.register_routes() - + # Initialize providers initialize_provider_map() initialize_tti_provider_map() @@ -152,7 +151,7 @@ def run_api( print("Starting Webscout OpenAI API server...") if port is None: port = DEFAULT_PORT - + AppConfig.set_config( api_key=None, default_provider=default_provider or AppConfig.default_provider, @@ -179,10 +178,10 @@ def run_api( print(f"Docs URL: {api_endpoint_base}/docs") # Show authentication status - print(f"Authentication: 🔓 DISABLED") + print("Authentication: 🔓 DISABLED") # Show rate limiting status - print(f"Rate Limiting: ⚡ DISABLED") + print("Rate Limiting: ⚡ DISABLED") print(f"Default Provider: {AppConfig.default_provider}") print(f"Workers: {workers}") @@ -265,14 +264,14 @@ def main(): args = parser.parse_args() # Print configuration summary - print(f"Configuration:") + print("Configuration:") print(f" Host: {args.host}") print(f" Port: {args.port}") print(f" Workers: {args.workers}") print(f" Log Level: {args.log_level}") print(f" Debug Mode: {args.debug}") - print(f" Authentication: 🔓 DISABLED") - print(f" Rate Limiting: ⚡ DISABLED") + print(" Authentication: 🔓 DISABLED") + print(" Rate Limiting: ⚡ DISABLED") print(f" Default Provider: {args.default_provider or 'Not set'}") print(f" Base URL: {args.base_url or 'Not set'}") print() diff --git a/webscout/server/ui_templates.py b/webscout/server/ui_templates.py index cefda687..60608c73 100644 --- a/webscout/server/ui_templates.py +++ b/webscout/server/ui_templates.py @@ -67,7 +67,7 @@ left: -50%; width: 200%; height: 200%; - background: + background: radial-gradient(ellipse at 20% 20%, rgba(99, 102, 241, 0.15) 0%, transparent 50%), radial-gradient(ellipse at 80% 20%, rgba(139, 92, 246, 0.12) 0%, transparent 50%), radial-gradient(ellipse at 50% 80%, rgba(6, 182, 212, 0.1) 0%, transparent 50%); @@ -699,7 +699,7 @@

Your All-in-One
AI Toolkit

- Access 90+ AI providers, multi-engine web search, text-to-speech, image generation, + Access 90+ AI providers, multi-engine web search, text-to-speech, image generation, and powerful developer tools — all through one unified, production-ready API.

@@ -1033,7 +1033,7 @@ font-weight: 600 !important; } -.swagger-ui table thead tr td, +.swagger-ui table thead tr td, .swagger-ui table thead tr th { color: var(--text-secondary) !important; border-bottom: 1px solid var(--border-color) !important; @@ -1088,8 +1088,8 @@ font-family: 'Consolas', monospace !important; } -.swagger-ui input, -.swagger-ui textarea, +.swagger-ui input, +.swagger-ui textarea, .swagger-ui select { background: #ffffff !important; color: var(--text-primary) !important; @@ -1099,8 +1099,8 @@ box-shadow: 0 1px 2px 0 rgba(0, 0, 0, 0.05) !important; } -.swagger-ui input:focus, -.swagger-ui textarea:focus, +.swagger-ui input:focus, +.swagger-ui textarea:focus, .swagger-ui select:focus { border-color: var(--primary-color) !important; outline: 2px solid rgba(79, 70, 229, 0.1) !important; diff --git a/webscout/swiftcli/__init__.py b/webscout/swiftcli/__init__.py index 1f3a9e08..e3ae63ef 100644 --- a/webscout/swiftcli/__init__.py +++ b/webscout/swiftcli/__init__.py @@ -18,46 +18,27 @@ """ from .core.cli import CLI -from .core.group import Group from .core.context import Context -from .plugins.base import Plugin -from .exceptions import ( - SwiftCLIException, - UsageError, - BadParameter, - ConfigError, - PluginError -) +from .core.group import Group # Command decorators -from .decorators.command import ( - command, - group, - argument, - flag, - pass_context -) +from .decorators.command import argument, command, flag, group, pass_context # Option decorators -from .decorators.options import ( - option, - envvar, - config_file, - version_option, - help_option -) +from .decorators.options import config_file, envvar, help_option, option, version_option # Output decorators from .decorators.output import ( - table_output, - progress, - panel_output, format_output, - pager_output, json_output, - yaml_output + pager_output, + panel_output, + progress, + table_output, + yaml_output, ) - +from .exceptions import BadParameter, ConfigError, PluginError, SwiftCLIException, UsageError +from .plugins.base import Plugin __all__ = [ # Core classes @@ -65,28 +46,28 @@ 'Group', 'Context', 'Plugin', - + # Exceptions 'SwiftCLIException', 'UsageError', 'BadParameter', 'ConfigError', 'PluginError', - + # Command decorators 'command', 'group', 'argument', 'flag', 'pass_context', - + # Option decorators 'option', 'envvar', 'config_file', 'version_option', 'help_option', - + # Output decorators 'table_output', 'progress', diff --git a/webscout/swiftcli/core/cli.py b/webscout/swiftcli/core/cli.py index d4e3670c..11e36c6d 100644 --- a/webscout/swiftcli/core/cli.py +++ b/webscout/swiftcli/core/cli.py @@ -1,28 +1,26 @@ """Main CLI application class.""" import sys -from typing import Any, Dict, List, Optional, Union - +from typing import Any, Dict, List, Optional from rich.console import Console -from .group import Group # Fix: Import Group for type checking and usage - from ..exceptions import UsageError from ..plugins.manager import PluginManager -from ..utils.formatting import format_error, format_success +from ..utils.formatting import format_error from .context import Context +from .group import Group # Fix: Import Group for type checking and usage console = Console() class CLI: """ Main CLI application class. - + The CLI class is the core of SwiftCLI. It handles command registration, argument parsing, and command execution. It also manages plugins and provides the main entry point for CLI applications. - + Attributes: name: Application name help: Application description @@ -31,7 +29,7 @@ class CLI: commands: Registered commands groups: Command groups plugin_manager: Plugin manager instance - + Example: >>> app = CLI(name="myapp", version="1.0.0") >>> @app.command() @@ -40,7 +38,7 @@ class CLI: ... print(f"Hello {name}!") >>> app.run() """ - + def __init__( self, name: str, @@ -50,7 +48,7 @@ def __init__( ): """ Initialize CLI application. - + Args: name: Application name help: Application description @@ -61,16 +59,16 @@ def __init__( self.help = help self.version = version self.debug = debug - + self.commands: Dict[str, Dict[str, Any]] = {} self.groups: Dict[str, 'Group'] = {} # type: ignore self.command_aliases: Dict[str, str] = {} self.command_chain: bool = False self.plugin_manager = PluginManager() - + # Initialize plugin manager with this CLI instance self.plugin_manager.init_plugins(self) - + def command( self, name: Optional[str] = None, @@ -80,13 +78,13 @@ def command( ): """ Decorator to register a command. - + Args: name: Command name (defaults to function name) help: Command help text aliases: Alternative command names hidden: Hide from help output - + Example: @app.command() def hello(name: str): @@ -102,14 +100,14 @@ def decorator(f): 'aliases': aliases or [], 'hidden': hidden } - + # Register aliases for alias in (aliases or []): self.commands[alias] = self.commands[cmd_name] - + return f return decorator - + def group( self, name: Optional[str] = None, @@ -118,25 +116,25 @@ def group( ): """ Create a command group. - + Args: name: Group name help: Group help text **kwargs: Additional group options - + Example: @app.group() def db(): '''Database commands''' pass - + @db.command() def migrate(): '''Run migrations''' pass """ # Group is now imported at the top - + def decorator(f): group_name = name or f.__name__ group = Group( @@ -148,15 +146,15 @@ def decorator(f): self.groups[group_name] = group return group return decorator - + def alias(self, command_name: str, alias_name: str) -> None: """ Add a command alias. - + Args: command_name: Original command name alias_name: Alias name - + Example: app.alias("list", "ls") """ @@ -164,69 +162,69 @@ def alias(self, command_name: str, alias_name: str) -> None: self.command_aliases[alias_name] = command_name else: raise UsageError(f"Command {command_name} not found") - + def enable_chaining(self, enabled: bool = True) -> None: """ Enable or disable command chaining. - + Args: enabled: Whether to enable chaining - + Example: app.enable_chaining(True) """ self.command_chain = enabled - + def run(self, args: Optional[List[str]] = None) -> int: """ Run the CLI application. - + Args: args: Command line arguments (defaults to sys.argv[1:]) - + Returns: Exit code (0 for success, non-zero for error) """ try: args = args or sys.argv[1:] - + # Show help if no arguments if not args or args[0] in ['-h', '--help']: self._print_help() return 0 - + # Show version if requested if args[0] in ['-v', '--version'] and self.version: console.print(self.version) return 0 - + command_name = args[0] command_args = args[1:] - + # Check if it's a command alias if command_name in self.command_aliases: command_name = self.command_aliases[command_name] - + # Check if it's a group command if command_name in self.groups: return self.groups[command_name].run(command_args) - + # Check if it's a regular command if command_name not in self.commands: format_error(f"Unknown command: {command_name}") self._print_help() return 1 - + # Create command context ctx = Context(self, command=command_name, debug=self.debug) - + # Run command through plugin system if not self.plugin_manager.before_command(command_name, command_args): return 1 - + try: - import inspect import asyncio + import inspect command = self.commands[command_name] func = command['func'] @@ -258,7 +256,7 @@ def run(self, args: Optional[List[str]] = None) -> int: raise format_error(str(e)) return 1 - + except KeyboardInterrupt: console.print("\nOperation cancelled by user") return 130 @@ -267,17 +265,17 @@ def run(self, args: Optional[List[str]] = None) -> int: raise format_error(str(e)) return 1 - + def generate_completion_script(self, shell: str = 'bash') -> str: """ Generate shell completion script. - + Args: shell: Shell type (bash, zsh, fish) - + Returns: Completion script as string - + Example: script = app.generate_completion_script('bash') print(script) @@ -290,7 +288,7 @@ def generate_completion_script(self, shell: str = 'bash') -> str: return self._generate_fish_completion() else: raise UsageError(f"Unsupported shell: {shell}") - + def _generate_bash_completion(self) -> str: """Generate bash completion script.""" commands = [] @@ -299,10 +297,10 @@ def _generate_bash_completion(self) -> str: commands.append(cmd_name) if 'aliases' in cmd: commands.extend(cmd['aliases']) - + for group_name in self.groups: commands.append(group_name) - + script = f"""#!/bin/bash # Bash completion for {self.name} @@ -317,7 +315,7 @@ def _generate_bash_completion(self) -> str: complete -F _{self.name}_completion {self.name} """ return script - + def _generate_zsh_completion(self) -> str: """Generate zsh completion script.""" commands = [] @@ -326,10 +324,10 @@ def _generate_zsh_completion(self) -> str: commands.append(cmd_name) if 'aliases' in cmd: commands.extend(cmd['aliases']) - + for group_name in self.groups: commands.append(group_name) - + script = f"""#compdef {self.name} local -a commands @@ -349,7 +347,7 @@ def _generate_zsh_completion(self) -> str: esac """ return script - + def _generate_fish_completion(self) -> str: """Generate fish completion script.""" commands = [] @@ -358,33 +356,36 @@ def _generate_fish_completion(self) -> str: commands.append(cmd_name) if 'aliases' in cmd: commands.extend(cmd['aliases']) - + for group_name in self.groups: commands.append(group_name) - + script = f"""# Fish completion for {self.name} complete -c {self.name} -n "__fish_use_subcommand" -a "{' '.join(commands)}" """ return script - + def _parse_args(self, command: Dict[str, Any], args: List[str]) -> Dict[str, Any]: """Parse command arguments.""" from ..utils.parsing import ( - parse_args, validate_required, convert_type, - validate_choice, get_env_var, validate_argument, - check_mutually_exclusive + check_mutually_exclusive, + convert_type, + get_env_var, + parse_args, + validate_argument, + validate_choice, ) - + params = {} func = command['func'] - + # Collect mutually exclusive groups exclusive_groups = [] - + # Parse command-line arguments parsed_args = parse_args(args) - + # Handle options if hasattr(func, '_options'): for opt in func._options: @@ -434,7 +435,7 @@ def _parse_args(self, command: Dict[str, Any], args: List[str]) -> Dict[str, Any # Flags are boolean; if a string value is provided, attempt to convert if isinstance(value_to_convert, str): # If 'type' is provided and is bool, convert accordingly - if 'type' in opt and opt['type'] == bool: + if 'type' in opt and opt['type'] is bool: value_to_convert = convert_type(value_to_convert, bool, name) else: value_to_convert = True @@ -454,20 +455,20 @@ def _parse_args(self, command: Dict[str, Any], args: List[str]) -> Dict[str, Any name, opt.get('case_sensitive', True) ) - + # Apply validation rules if opt.get('validation'): value_to_convert = validate_argument( - str(value_to_convert), - opt['validation'], + str(value_to_convert), + opt['validation'], name ) - + params[name] = value_to_convert # Apply callback if provided if opt.get('callback') and callable(opt.get('callback')): params[name] = opt.get('callback')(params[name]) - + # Collect mutually exclusive groups if opt.get('mutually_exclusive'): exclusive_groups.append(opt['mutually_exclusive']) @@ -475,7 +476,7 @@ def _parse_args(self, command: Dict[str, Any], args: List[str]) -> Dict[str, Any raise UsageError(f"Missing required option: {name}") elif 'default' in opt: params[name] = opt['default'] - + # Handle arguments if hasattr(func, '_arguments'): for i, arg in enumerate(func._arguments): @@ -484,13 +485,13 @@ def _parse_args(self, command: Dict[str, Any], args: List[str]) -> Dict[str, Any value = parsed_args[f'arg{i}'] if 'type' in arg: value = convert_type(value, arg['type'], name) - + # Apply validation rules if arg.get('validation'): value = validate_argument(value, arg['validation'], name) - + params[name] = value - + # Collect mutually exclusive groups if arg.get('mutually_exclusive'): exclusive_groups.append(arg['mutually_exclusive']) @@ -498,11 +499,11 @@ def _parse_args(self, command: Dict[str, Any], args: List[str]) -> Dict[str, Any raise UsageError(f"Missing required argument: {name}") elif 'default' in arg: params[name] = arg['default'] - + # Check mutually exclusive options if exclusive_groups: check_mutually_exclusive(params, exclusive_groups) - + # Handle environment variables if hasattr(func, '_envvars'): for env in func._envvars: @@ -515,15 +516,15 @@ def _parse_args(self, command: Dict[str, Any], args: List[str]) -> Dict[str, Any ) if value is not None: params[name] = value - + return params - + def _print_help(self) -> None: """Print application help message.""" console.print(f"\n[bold]{self.name}[/]") if self.help: console.print(f"\n{self.help}") - + # Show commands console.print("\n[bold]Commands:[/]") printed = set() @@ -536,7 +537,7 @@ def _print_help(self) -> None: aliases = cmd.get('aliases', []) alias_text = f" (aliases: {', '.join(aliases)})" if aliases else "" console.print(f" {primary:20} {cmd['help'] or ''}{alias_text}") - + # Show command groups for name, group in self.groups.items(): console.print(f"\n[bold]{name} commands:[/]") @@ -565,10 +566,10 @@ def _print_help(self) -> None: if not hidden: alias_text = f" (aliases: {', '.join(aliases)})" if aliases else "" console.print(f" {primary:20} {help_text}{alias_text}") - + console.print("\nUse -h or --help with any command for more info") if self.version: console.print("Use -v or --version to show version") - + def __repr__(self) -> str: return f"" diff --git a/webscout/swiftcli/core/context.py b/webscout/swiftcli/core/context.py index 846448c2..b356c2fd 100644 --- a/webscout/swiftcli/core/context.py +++ b/webscout/swiftcli/core/context.py @@ -1,11 +1,15 @@ """Context handling for SwiftCLI.""" -from typing import Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Dict, Optional + +if TYPE_CHECKING: + from .cli import CLI + class Context: """ Context object that holds state for the CLI app. - + The Context class provides access to the CLI application instance and maintains state throughout command execution. It can be used to pass data between commands and access global configuration. @@ -17,7 +21,7 @@ class Context: obj: An object that can be used to store arbitrary data. params: Dictionary of current command parameters. debug: Debug mode flag. - + Example: @app.command() @pass_context @@ -26,7 +30,7 @@ def status(ctx): print(f"App: {ctx.cli.name}") print(f"Debug: {ctx.debug}") """ - + def __init__( self, cli: 'CLI', # type: ignore @@ -41,34 +45,34 @@ def __init__( self.obj = obj self.params: Dict[str, Any] = {} self.debug = debug - + def get_parameter(self, name: str, default: Any = None) -> Any: """ Get a parameter value from the context. - + Args: name: Parameter name default: Default value if parameter not found - + Returns: Parameter value or default """ return self.params.get(name, default) - + def set_parameter(self, name: str, value: Any) -> None: """ Set a parameter value in the context. - + Args: name: Parameter name value: Parameter value """ self.params[name] = value - + def get_parent_context(self) -> Optional['Context']: """Get the parent context if it exists.""" return self.parent - + def create_child_context( self, command: Optional[str] = None, @@ -76,11 +80,11 @@ def create_child_context( ) -> 'Context': """ Create a new child context. - + Args: command: Command name for the child context obj: Object to store in child context - + Returns: New child context """ @@ -91,7 +95,7 @@ def create_child_context( obj=obj, debug=self.debug ) - + @property def root_context(self) -> 'Context': """Get the root context by traversing up the parent chain.""" @@ -99,6 +103,6 @@ def root_context(self) -> 'Context': while ctx.parent is not None: ctx = ctx.parent return ctx - + def __repr__(self) -> str: return f"" diff --git a/webscout/swiftcli/core/group.py b/webscout/swiftcli/core/group.py index 89366e41..34314896 100644 --- a/webscout/swiftcli/core/group.py +++ b/webscout/swiftcli/core/group.py @@ -1,10 +1,9 @@ """Command group handling for SwiftCLI.""" -from typing import Any, Dict, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Optional from rich.console import Console -from ..exceptions import UsageError from ..utils.formatting import format_error from .context import Context @@ -16,10 +15,10 @@ class Group: """ Command group that can contain subcommands. - + Groups allow organizing related commands together and support command chaining for building command pipelines. - + Attributes: name: Group name help: Group description @@ -27,19 +26,19 @@ class Group: parent: Parent CLI instance chain: Enable command chaining invoke_without_command: Allow invoking group without subcommand - + Example: @app.group() def db(): '''Database commands''' pass - + @db.command() def migrate(): '''Run database migrations''' print("Running migrations...") """ - + def __init__( self, name: str, @@ -50,7 +49,7 @@ def __init__( ): """ Initialize command group. - + Args: name: Group name help: Group description @@ -64,7 +63,7 @@ def __init__( self.chain = chain self.invoke_without_command = invoke_without_command self.commands: Dict[str, Dict[str, Any]] = {} - + def command( self, name: Optional[str] = None, @@ -74,13 +73,13 @@ def command( ): """ Decorator to register a command in this group. - + Args: name: Command name (defaults to function name) help: Command help text aliases: Alternative command names hidden: Hide from help output - + Example: @group.command() def status(): @@ -96,14 +95,14 @@ def decorator(f): 'aliases': aliases or [], 'hidden': hidden } - + # Register aliases for alias in (aliases or []): self.commands[alias] = self.commands[cmd_name] - + return f return decorator - + def group( self, name: Optional[str] = None, @@ -112,12 +111,12 @@ def group( ): """ Create a subgroup within this group. - + Args: name: Subgroup name help: Subgroup help text **kwargs: Additional group options - + Example: @group.group() def config(): @@ -134,14 +133,14 @@ def decorator(f): self.commands[subgroup.name] = subgroup return subgroup return decorator - + def run(self, args: List[str]) -> int: """ Run a command in this group. - + Args: args: Command arguments - + Returns: Exit code (0 for success, non-zero for error) """ @@ -150,39 +149,39 @@ def run(self, args: List[str]) -> int: if not args or args[0] in ['-h', '--help']: self._print_help() return 0 - + command_name = args[0] command_args = args[1:] - + # Check if command exists if command_name not in self.commands: format_error(f"Unknown command: {self.name} {command_name}") self._print_help() return 1 - + command = self.commands[command_name] - + # Handle nested groups if isinstance(command, Group): return command.run(command_args) - + # Create command context ctx = Context( self.parent, command=f"{self.name} {command_name}", debug=getattr(self.parent, 'debug', False) ) - + # Run command through plugin system if self.parent and not self.parent.plugin_manager.before_command( f"{self.name} {command_name}", command_args ): return 1 - + try: - import inspect import asyncio + import inspect func = command['func'] params = self._parse_args(command, command_args) @@ -202,20 +201,20 @@ def run(self, args: List[str]) -> int: # If function returned a coroutine-like object if not inspect.iscoroutine(result) and hasattr(result, '__await__'): result = asyncio.run(result) - + if self.parent: self.parent.plugin_manager.after_command( f"{self.name} {command_name}", command_args, result ) - + # Handle command chaining if self.chain and result is not None: return result - + return 0 - + except Exception as e: if self.parent: self.parent.plugin_manager.on_error( @@ -226,27 +225,27 @@ def run(self, args: List[str]) -> int: raise format_error(str(e)) return 1 - + except Exception as e: if getattr(self.parent, 'debug', False): raise format_error(str(e)) return 1 - + def _parse_args(self, command: Dict[str, Any], args: List[str]) -> Dict[str, Any]: """Parse command arguments.""" # Use parent CLI's argument parser if available if self.parent: return self.parent._parse_args(command, args) - + # Fallback to basic argument parsing from ..utils.parsing import parse_args return parse_args(args) - + def _print_help(self) -> None: """Print group help message.""" console.print(f"\n[bold]{self.name}[/] - {self.help or ''}") - + console.print("\n[bold]Commands:[/]") printed = set() for name, cmd in self.commands.items(): @@ -268,8 +267,8 @@ def _print_help(self) -> None: aliases = cmd.get('aliases', []) alias_text = f" (aliases: {', '.join(aliases)})" if aliases else "" console.print(f" {primary:20} {cmd['help'] or ''}{alias_text}") - + console.print("\nUse -h or --help with any command for more info") - + def __repr__(self) -> str: return f"" diff --git a/webscout/swiftcli/decorators/__init__.py b/webscout/swiftcli/decorators/__init__.py index bc574ae7..90a55ad5 100644 --- a/webscout/swiftcli/decorators/__init__.py +++ b/webscout/swiftcli/decorators/__init__.py @@ -1,8 +1,8 @@ """Decorators for SwiftCLI.""" -from .command import command, group, argument, flag, pass_context -from .options import option, envvar, config_file, version_option, help_option -from .output import table_output, progress, panel_output, format_output, pager_output +from .command import argument, command, flag, group, pass_context +from .options import config_file, envvar, help_option, option, version_option +from .output import format_output, pager_output, panel_output, progress, table_output __all__ = [ # Command decorators @@ -11,14 +11,14 @@ 'argument', 'flag', 'pass_context', - + # Option decorators 'option', 'envvar', 'config_file', 'version_option', 'help_option', - + # Output decorators 'table_output', 'progress', diff --git a/webscout/swiftcli/decorators/command.py b/webscout/swiftcli/decorators/command.py index 748df266..44ce4458 100644 --- a/webscout/swiftcli/decorators/command.py +++ b/webscout/swiftcli/decorators/command.py @@ -1,10 +1,11 @@ """Command decorators for SwiftCLI.""" from functools import wraps -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, List, Optional from ..core.context import Context + def command( name: str = None, help: str = None, @@ -13,21 +14,21 @@ def command( ) -> Callable: """ Decorator to register a new command. - + This decorator marks a function as a CLI command and provides metadata about how the command should be registered and displayed. - + Args: name: Command name (defaults to function name) help: Help text (defaults to function docstring) aliases: Alternative names for the command hidden: Whether to hide from help output - + Example: @command(name="greet", help="Say hello") def hello(name: str): print(f"Hello {name}!") - + @command(aliases=["hi", "hey"]) def hello(name: str): '''Say hello to someone''' @@ -51,31 +52,31 @@ def group( ) -> Callable: """ Decorator to create a command group. - + Command groups can contain subcommands and optionally chain their results. - + Args: name: Group name (defaults to function name) help: Help text (defaults to function docstring) chain: Whether to chain command results invoke_without_command: Allow group to be invoked without subcommand - + Example: @group() def db(): '''Database commands''' pass - + @db.command() def migrate(): '''Run database migrations''' print("Running migrations...") - + @group(chain=True) def process(): '''Process data''' pass - + @process.command() def validate(): '''Validate data''' @@ -94,10 +95,10 @@ def decorator(f: Callable) -> Callable: def pass_context(f: Callable) -> Callable: """ Decorator to pass CLI context to command. - + This decorator injects the current Context object as the first argument to the decorated command function. - + Example: @command() @pass_context @@ -112,17 +113,17 @@ def status(ctx): def completion(func: Optional[Callable] = None) -> Callable: """ Decorator to provide shell completion for a command. - + The decorated function should return a list of possible completions based on the current incomplete value. - + Example: @command() @option("--service", type=str) def restart(service: str): '''Restart a service''' print(f"Restarting {service}...") - + @restart.completion() def complete_service(ctx, incomplete): services = ["nginx", "apache", "mysql"] @@ -136,7 +137,7 @@ def wrapper(ctx: Context, incomplete: str) -> List[str]: except Exception: return [] return wrapper - + if func is None: return decorator return decorator(func) @@ -152,9 +153,9 @@ def argument( ) -> Callable: """ Decorator to add a command argument. - + Arguments are positional parameters that must be provided in order. - + Args: name: Argument name type: Expected type @@ -163,7 +164,7 @@ def argument( default: Default value if not required validation: Dictionary of validation rules (min_length, max_length, pattern, choices, etc.) mutually_exclusive: List of argument names that are mutually exclusive with this argument - + Example: @command() @argument("name", validation={{'min_length': 2, 'max_length': 50}}) @@ -176,7 +177,7 @@ def greet(name: str, count: int): def decorator(f: Callable) -> Callable: if not hasattr(f, '_arguments'): f._arguments = [] - + f._arguments.append({ 'name': name, 'type': type, @@ -196,15 +197,15 @@ def flag( ) -> Callable: """ Decorator to add a boolean flag option. - + Flags are special options that don't take a value - they're either present (True) or absent (False). - + Args: name: Flag name help: Help text hidden: Whether to hide from help output - + Example: @command() @flag("--verbose", help="Enable verbose output") @@ -216,7 +217,7 @@ def process(verbose: bool): def decorator(f: Callable) -> Callable: if not hasattr(f, '_options'): f._options = [] - + f._options.append({ 'param_decls': [name], 'is_flag': True, diff --git a/webscout/swiftcli/decorators/options.py b/webscout/swiftcli/decorators/options.py index 450900a9..cd0100b6 100644 --- a/webscout/swiftcli/decorators/options.py +++ b/webscout/swiftcli/decorators/options.py @@ -28,9 +28,9 @@ def option( ) -> Callable: """ Decorator to add an option to a command. - + Options are named parameters that can be provided in any order. - + Args: param_decls: Option names (e.g., "--name", "-n") type: Expected type @@ -51,7 +51,7 @@ def option( hidden: Whether to hide from help output validation: Dictionary of validation rules (min_length, max_length, pattern, etc.) mutually_exclusive: List of option names that are mutually exclusive with this option - + Example: @command() @option("--count", "-c", type=int, default=1) @@ -66,7 +66,7 @@ def process(count: int, format: str, verbose: bool, name: str): def decorator(f: Callable) -> Callable: if not hasattr(f, '_options'): f._options = [] - + f._options.append({ 'param_decls': param_decls, 'type': type, @@ -101,14 +101,14 @@ def envvar( ) -> Callable: """ Decorator to load option value from environment variable. - + Args: name: Environment variable name type: Expected type required: Whether variable is required default: Default value if not set help: Help text - + Example: @command() @envvar("API_KEY", required=True) @@ -120,7 +120,7 @@ def api_call(api_key: str, api_url: str): def decorator(f: Callable) -> Callable: if not hasattr(f, '_envvars'): f._envvars = [] - + f._envvars.append({ 'name': name, 'type': type, @@ -140,21 +140,21 @@ def config_file( ) -> Callable: """ Decorator to load configuration from file. - + Args: path: Config file path section: Config section to load required: Whether config is required auto_create: Whether to create file if missing format: File format (json, yaml, ini) - + Example: @command() @config_file("~/.myapp/config.json") def setup(config: dict): '''Setup application''' print(f"Database: {config.get('database')}") - + @command() @config_file("config.ini", section="api") def api(config: dict): @@ -180,13 +180,13 @@ def version_option( ) -> Callable: """ Decorator to add version option to command. - + Args: version: Version string prog_name: Program name message: Custom version message package_name: Package name to get version from - + Example: @command() @version_option(version="1.0.0") @@ -210,11 +210,11 @@ def help_option( ) -> Callable: """ Decorator to customize help option. - + Args: param_decls: Help option flags help: Help text - + Example: @command() @help_option(["--help", "-h"], "Show this message") diff --git a/webscout/swiftcli/decorators/output.py b/webscout/swiftcli/decorators/output.py index a6adcbb1..290e8854 100644 --- a/webscout/swiftcli/decorators/output.py +++ b/webscout/swiftcli/decorators/output.py @@ -1,33 +1,33 @@ """Output formatting decorators for SwiftCLI.""" import json -import yaml from functools import wraps -from typing import Any, Callable, List, Optional, Union +from typing import Callable, List, Optional, Union +import yaml from rich.console import Console -from rich.table import Table from rich.panel import Panel +from rich.table import Table # Handle different versions of rich try: from rich.progress import ( + BarColumn, Progress, SpinnerColumn, - TextColumn, - BarColumn, TaskProgressColumn, - TimeRemainingColumn + TextColumn, + TimeRemainingColumn, ) except ImportError: # Fallback for older versions of rich try: from rich.progress import ( + BarColumn, Progress, SpinnerColumn, TextColumn, - BarColumn, - TimeRemainingColumn + TimeRemainingColumn, ) # Create a simple TaskProgressColumn replacement for older versions class TaskProgressColumn: @@ -74,14 +74,14 @@ def table_output( ) -> Callable: """ Decorator to format command output as a table. - + Args: headers: Column headers title: Table title style: Table style show_lines: Show row/column lines expand: Expand table to terminal width - + Example: @command() @table_output(["ID", "Name", "Status"]) @@ -104,15 +104,15 @@ def wrapper(*args, **kwargs): show_lines=show_lines, expand=expand ) - + # Add columns for header in headers: table.add_column(header) - + # Add rows for row in result: table.add_row(*[str(cell) for cell in row]) - + console.print(table) return result return wrapper @@ -125,12 +125,12 @@ def json_output( ) -> Callable: """ Decorator to format command output as JSON. - + Args: indent: Indentation level for pretty printing sort_keys: Whether to sort dictionary keys ensure_ascii: Whether to escape non-ASCII characters - + Example: @command() @json_output(indent=2) @@ -160,11 +160,11 @@ def yaml_output( ) -> Callable: """ Decorator to format command output as YAML. - + Args: default_flow_style: Whether to use flow style for collections sort_keys: Whether to sort dictionary keys - + Example: @command() @yaml_output() @@ -197,10 +197,10 @@ def progress( ) -> Callable: """ Decorator to show progress for long-running commands. - + The decorated function should be a generator that yields progress updates. - + Args: description: Task description total: Total number of steps @@ -208,7 +208,7 @@ def progress( show_bar: Show progress bar show_percentage: Show percentage complete show_time: Show time remaining - + Example: @command() @progress("Processing files") @@ -224,24 +224,24 @@ def wrapper(*args, **kwargs): columns = [] columns.append(SpinnerColumn()) columns.append(TextColumn("[progress.description]{task.description}")) - + if show_bar: columns.append(BarColumn()) if show_percentage: try: columns.append(TaskProgressColumn()) - except: + except Exception: # Fallback for older rich versions columns.append(TextColumn("[progress.percentage]{task.percentage:>3.0f}%")) if show_time: columns.append(TimeRemainingColumn()) - + with Progress(*columns, transient=transient) as progress: task = progress.add_task( description or f.__name__, total=total ) - + try: for update in f(*args, **kwargs): if isinstance(update, (int, float)): @@ -261,7 +261,7 @@ def wrapper(*args, **kwargs): raise finally: progress.update(task, completed=total or 100) - + return wrapper return decorator @@ -273,13 +273,13 @@ def panel_output( ) -> Callable: """ Decorator to display command output in a panel. - + Args: title: Panel title style: Panel style expand: Expand panel to terminal width padding: Panel padding - + Example: @command() @panel_output(title="System Status") @@ -310,11 +310,11 @@ def format_output( ) -> Callable: """ Decorator to format command output using a template. - + Args: template: Format string template style: Rich style string - + Example: @command() @format_output("Created user {name} with ID {id}") @@ -331,7 +331,7 @@ def wrapper(*args, **kwargs): output = template.format(**result) else: output = template.format(result) - + if style: console.print(output, style=style) else: @@ -346,11 +346,11 @@ def pager_output( ) -> Callable: """ Decorator to display command output in a pager. - + Args: use_pager: Whether to use pager style: Rich style string - + Example: @command() @pager_output() diff --git a/webscout/swiftcli/plugins/base.py b/webscout/swiftcli/plugins/base.py index cb78d3ba..b47c3086 100644 --- a/webscout/swiftcli/plugins/base.py +++ b/webscout/swiftcli/plugins/base.py @@ -2,134 +2,133 @@ from typing import Any, Dict, List, Optional -from ..exceptions import PluginError class Plugin: """ Base class for SwiftCLI plugins. - + Plugins can extend the CLI functionality by hooking into various stages of command execution. Subclass this class and override the methods you want to hook into. - + Attributes: app: The CLI application instance enabled: Whether the plugin is enabled config: Plugin configuration dictionary - + Example: class LoggingPlugin(Plugin): def before_command(self, command, args): print(f"[LOG] Running command: {command}") - + def after_command(self, command, args, result): print(f"[LOG] Command completed: {result}") """ - + def __init__(self): self.app = None # Set by plugin manager self.enabled: bool = True self.config: Dict[str, Any] = {} - + def init_app(self, app: Any) -> None: """ Initialize plugin with CLI app instance. - + Args: app: The CLI application instance """ self.app = app - + def before_command(self, command: str, args: List[str]) -> Optional[bool]: """ Called before command execution. - + Args: command: Command name args: Command arguments - + Returns: Optional[bool]: Return False to prevent command execution """ pass - + def after_command(self, command: str, args: List[str], result: Any) -> None: """ Called after command execution. - + Args: command: Command name args: Command arguments result: Command result """ pass - + def on_error(self, command: str, error: Exception) -> None: """ Called when command raises an error. - + Args: command: Command name error: The exception that was raised """ pass - + def on_help(self, command: str) -> Optional[str]: """ Called when help is requested for a command. - + Args: command: Command name - + Returns: Optional[str]: Additional help text to display """ pass - + def on_completion(self, command: str, incomplete: str) -> List[str]: """ Called when shell completion is requested. - + Args: command: Command name incomplete: Incomplete text to complete - + Returns: List[str]: Possible completions """ return [] - + def configure(self, config: Dict[str, Any]) -> None: """ Configure the plugin. - + Args: config: Configuration dictionary """ self.config.update(config) - + def validate_config(self) -> None: """ Validate plugin configuration. - + Raises: PluginError: If configuration is invalid """ pass - + def enable(self) -> None: """Enable the plugin.""" self.enabled = True - + def disable(self) -> None: """Disable the plugin.""" self.enabled = False - + @property def name(self) -> str: """Get plugin name.""" return self.__class__.__name__ - + def __repr__(self) -> str: status = "enabled" if self.enabled else "disabled" return f"<{self.name} [{status}]>" diff --git a/webscout/swiftcli/plugins/manager.py b/webscout/swiftcli/plugins/manager.py index 8dae763e..31801f01 100644 --- a/webscout/swiftcli/plugins/manager.py +++ b/webscout/swiftcli/plugins/manager.py @@ -5,7 +5,7 @@ import sys import tempfile from pathlib import Path -from typing import Any, Dict, List, Optional, Type +from typing import Any, Dict, List, Optional from rich.console import Console @@ -17,28 +17,28 @@ class PluginManager: """ Manages SwiftCLI plugins. - + The plugin manager handles plugin registration, loading, and execution. It provides hooks for plugins to extend CLI functionality at various points during command execution. - + Attributes: plugins: List of registered plugins plugin_dir: Directory where plugins are stored - + Example: # Register a plugin plugin_manager = PluginManager() plugin_manager.register(LoggingPlugin()) - + # Load plugins from directory plugin_manager.load_plugins() """ - + def __init__(self, plugin_dir: Optional[str] = None): """ Initialize plugin manager. - + Args: plugin_dir: Optional custom plugin directory path """ @@ -51,112 +51,112 @@ def __init__(self, plugin_dir: Optional[str] = None): # Create a temporary directory that will be cleaned up when the process exits self.temp_dir = tempfile.TemporaryDirectory(prefix="swiftcli_") self.plugin_dir = self.temp_dir.name - + # Add plugin directory to Python path if self.plugin_dir not in sys.path: sys.path.append(self.plugin_dir) - + def register(self, plugin: Plugin) -> None: """ Register a new plugin. - + Args: plugin: Plugin instance to register - + Raises: PluginError: If plugin is invalid or already registered """ if not isinstance(plugin, Plugin): raise PluginError(f"Invalid plugin type: {type(plugin)}") - + if self._get_plugin(plugin.name): raise PluginError(f"Plugin already registered: {plugin.name}") - + try: plugin.validate_config() except Exception as e: raise PluginError(f"Plugin configuration invalid: {str(e)}") - + self.plugins.append(plugin) - + def unregister(self, plugin_name: str) -> None: """ Unregister a plugin. - + Args: plugin_name: Name of plugin to unregister - + Raises: PluginError: If plugin not found """ plugin = self._get_plugin(plugin_name) if not plugin: raise PluginError(f"Plugin not found: {plugin_name}") - + self.plugins.remove(plugin) - + def load_plugins(self) -> None: """ Load all plugins from plugin directory. - + This method searches for Python files in the plugin directory and attempts to load any Plugin subclasses defined in them. """ for file in Path(self.plugin_dir).glob("*.py"): if file.name.startswith("_"): continue - + try: module = importlib.import_module(file.stem) - + # Find Plugin subclasses in module for attr_name in dir(module): attr = getattr(module, attr_name) - if (isinstance(attr, type) and - issubclass(attr, Plugin) and + if (isinstance(attr, type) and + issubclass(attr, Plugin) and attr is not Plugin): plugin = attr() self.register(plugin) - + except Exception as e: console.print(f"[red]Error loading plugin {file.name}: {e}[/red]") - + def init_plugins(self, app: Any) -> None: """ Initialize all plugins with the CLI application instance. - + Args: app: The CLI application instance """ for plugin in self.plugins: plugin.init_app(app) - + def configure_plugins(self, config: Dict[str, Dict[str, Any]]) -> None: """ Configure plugins with provided configuration. - + Args: config: Dictionary of plugin configurations """ for plugin in self.plugins: if plugin.name in config: plugin.configure(config[plugin.name]) - + def before_command(self, command: str, args: List[str]) -> bool: """ Run before_command hooks for all plugins. - + Args: command: Command name args: Command arguments - + Returns: bool: False if any plugin prevents command execution """ for plugin in self.plugins: if not plugin.enabled: continue - + try: result = plugin.before_command(command, args) if result is False: @@ -166,13 +166,13 @@ def before_command(self, command: str, args: List[str]) -> bool: if plugin.app and getattr(plugin.app, 'debug', False): import traceback traceback.print_exc() - + return True - + def after_command(self, command: str, args: List[str], result: Any) -> None: """ Run after_command hooks for all plugins. - + Args: command: Command name args: Command arguments @@ -181,7 +181,7 @@ def after_command(self, command: str, args: List[str], result: Any) -> None: for plugin in self.plugins: if not plugin.enabled: continue - + try: plugin.after_command(command, args, result) except Exception as e: @@ -189,11 +189,11 @@ def after_command(self, command: str, args: List[str], result: Any) -> None: if plugin.app and getattr(plugin.app, 'debug', False): import traceback traceback.print_exc() - + def on_error(self, command: str, error: Exception) -> None: """ Run error hooks for all plugins. - + Args: command: Command name error: The exception that was raised @@ -201,7 +201,7 @@ def on_error(self, command: str, error: Exception) -> None: for plugin in self.plugins: if not plugin.enabled: continue - + try: plugin.on_error(command, error) except Exception as e: @@ -209,14 +209,14 @@ def on_error(self, command: str, error: Exception) -> None: if plugin.app and getattr(plugin.app, 'debug', False): import traceback traceback.print_exc() - + def get_help_text(self, command: str) -> List[str]: """ Get additional help text from plugins. - + Args: command: Command name - + Returns: List[str]: List of help text strings from plugins """ @@ -224,24 +224,24 @@ def get_help_text(self, command: str) -> List[str]: for plugin in self.plugins: if not plugin.enabled: continue - + try: help_text = plugin.on_help(command) if help_text: help_texts.append(help_text) except Exception as e: console.print(f"[red]Error in plugin {plugin.name}: {e}[/red]") - + return help_texts - + def get_completions(self, command: str, incomplete: str) -> List[str]: """ Get command completions from plugins. - + Args: command: Command name incomplete: Incomplete text to complete - + Returns: List[str]: Combined list of completions from all plugins """ @@ -249,21 +249,21 @@ def get_completions(self, command: str, incomplete: str) -> List[str]: for plugin in self.plugins: if not plugin.enabled: continue - + try: plugin_completions = plugin.on_completion(command, incomplete) completions.extend(plugin_completions) except Exception as e: console.print(f"[red]Error in plugin {plugin.name}: {e}[/red]") - + return list(set(completions)) # Remove duplicates - + def _get_plugin(self, name: str) -> Optional[Plugin]: """Get plugin by name.""" for plugin in self.plugins: if plugin.name == name: return plugin return None - + def __repr__(self) -> str: return f"" diff --git a/webscout/swiftcli/utils/__init__.py b/webscout/swiftcli/utils/__init__.py index d036364c..8a60bb1e 100644 --- a/webscout/swiftcli/utils/__init__.py +++ b/webscout/swiftcli/utils/__init__.py @@ -1,32 +1,31 @@ """Utility functions for SwiftCLI.""" from .formatting import ( - style_text, - format_error, - format_warning, - format_success, - format_info, + clear_screen, + create_padding, create_table, - truncate_text, - wrap_text, format_dict, + format_error, + format_info, format_list, - strip_ansi, + format_success, + format_warning, get_terminal_size, - clear_screen, - create_padding + strip_ansi, + style_text, + truncate_text, + wrap_text, ) - from .parsing import ( - parse_args, - validate_required, convert_type, - validate_choice, + get_env_var, load_config_file, + parse_args, + parse_dict, parse_key_value, parse_list, - parse_dict, - get_env_var + validate_choice, + validate_required, ) __all__ = [ @@ -45,7 +44,7 @@ 'get_terminal_size', 'clear_screen', 'create_padding', - + # Parsing utilities 'parse_args', 'validate_required', diff --git a/webscout/swiftcli/utils/formatting.py b/webscout/swiftcli/utils/formatting.py index fcc62062..8738cdc7 100644 --- a/webscout/swiftcli/utils/formatting.py +++ b/webscout/swiftcli/utils/formatting.py @@ -4,10 +4,9 @@ from typing import Any, Dict, List, Optional, Union from rich.console import Console -from rich.style import Style -from rich.text import Text -from rich.table import Table from rich.padding import Padding +from rich.table import Table +from rich.text import Text console = Console() @@ -20,14 +19,14 @@ def style_text( ) -> Text: """ Apply styling to text. - + Args: text: Text to style color: Text color bold: Bold text italic: Italic text underline: Underline text - + Returns: Rich Text object with applied styling """ @@ -40,13 +39,13 @@ def style_text( style.append("italic") if underline: style.append("underline") - + return Text(text, style=" ".join(style)) def format_error(message: str, title: str = "Error") -> None: """ Format and display error message. - + Args: message: Error message title: Error title @@ -56,7 +55,7 @@ def format_error(message: str, title: str = "Error") -> None: def format_warning(message: str, title: str = "Warning") -> None: """ Format and display warning message. - + Args: message: Warning message title: Warning title @@ -66,7 +65,7 @@ def format_warning(message: str, title: str = "Warning") -> None: def format_success(message: str, title: str = "Success") -> None: """ Format and display success message. - + Args: message: Success message title: Success title @@ -76,7 +75,7 @@ def format_success(message: str, title: str = "Success") -> None: def format_info(message: str, title: str = "Info") -> None: """ Format and display info message. - + Args: message: Info message title: Info title @@ -92,14 +91,14 @@ def create_table( ) -> Table: """ Create a formatted table. - + Args: headers: Column headers rows: Table rows title: Table title style: Table style show_lines: Show row/column lines - + Returns: Rich Table object """ @@ -109,15 +108,15 @@ def create_table( header_style="bold blue", show_lines=show_lines ) - + # Add columns for header in headers: table.add_column(header) - + # Add rows for row in rows: table.add_row(*[str(cell) for cell in row]) - + return table def truncate_text( @@ -127,12 +126,12 @@ def truncate_text( ) -> str: """ Truncate text to specified length. - + Args: text: Text to truncate max_length: Maximum length suffix: Truncation suffix - + Returns: Truncated text """ @@ -148,13 +147,13 @@ def wrap_text( ) -> str: """ Wrap text to specified width. - + Args: text: Text to wrap width: Maximum line width indent: Indentation for wrapped lines initial_indent: Indentation for first line - + Returns: Wrapped text """ @@ -173,12 +172,12 @@ def format_dict( ) -> str: """ Format dictionary for display. - + Args: data: Dictionary to format indent: Indentation level sort_keys: Sort dictionary keys - + Returns: Formatted string """ @@ -197,12 +196,12 @@ def format_list( ) -> str: """ Format list for display. - + Args: items: List to format bullet: Bullet point character indent: Indentation level - + Returns: Formatted string """ @@ -212,10 +211,10 @@ def format_list( def strip_ansi(text: str) -> str: """ Remove ANSI escape sequences from text. - + Args: text: Text containing ANSI sequences - + Returns: Clean text """ @@ -225,7 +224,7 @@ def strip_ansi(text: str) -> str: def get_terminal_size() -> tuple: """ Get terminal size. - + Returns: Tuple of (width, height) """ @@ -241,11 +240,11 @@ def create_padding( ) -> Padding: """ Add padding around content. - + Args: renderable: Content to pad pad: Padding amount - + Returns: Padded content """ diff --git a/webscout/swiftcli/utils/parsing.py b/webscout/swiftcli/utils/parsing.py index 6f1771b8..faba30da 100644 --- a/webscout/swiftcli/utils/parsing.py +++ b/webscout/swiftcli/utils/parsing.py @@ -1,24 +1,26 @@ """Utility functions for parsing and validating command-line arguments.""" +import json import os import re -import json -import yaml from pathlib import Path -from typing import Any, Dict, List, Optional, Union, Type, Pattern +from typing import Any, Dict, List, Type, Union + +import yaml from ..exceptions import BadParameter, UsageError + def parse_args(args: List[str]) -> Dict[str, Any]: """ Parse command line arguments into a dictionary. - + Args: args: List of command line arguments - + Returns: Dictionary of parsed arguments - + Example: >>> parse_args(['--name', 'test', '--flag', '-n', '42']) {'name': 'test', 'flag': True, 'n': '42'} @@ -27,7 +29,7 @@ def parse_args(args: List[str]) -> Dict[str, Any]: i = 0 while i < len(args): arg = args[i] - + # Handle flags/options if arg.startswith('-'): # Support --key=value or -k=value syntax @@ -57,9 +59,9 @@ def parse_args(args: List[str]) -> Dict[str, Any]: # Positional argument pos_index = len([k for k in parsed.keys() if k.startswith('arg')]) parsed[f'arg{pos_index}'] = arg - + i += 1 - + return parsed def validate_required( @@ -68,11 +70,11 @@ def validate_required( ) -> None: """ Validate required parameters are present. - + Args: params: Parameter dictionary required: List of required parameter names - + Raises: UsageError: If required parameter is missing """ @@ -87,21 +89,21 @@ def convert_type( ) -> Any: """ Convert string value to specified type. - + Args: value: String value to convert type_: Target type param_name: Parameter name for error messages - + Returns: Converted value - + Raises: BadParameter: If conversion fails """ try: # Handle boolean conversion robustly when the input may already be a bool - if type_ == bool: + if type_ is bool: if isinstance(value, bool): return value if isinstance(value, (int, float)): @@ -129,13 +131,13 @@ def validate_choice( ) -> None: """ Validate value is one of allowed choices. - + Args: value: Value to validate choices: List of allowed choices param_name: Parameter name for error messages case_sensitive: Whether to do case-sensitive comparison - + Raises: BadParameter: If value not in choices """ @@ -158,31 +160,31 @@ def validate_argument( ) -> str: """ Validate argument against validation rules. - + Args: value: Argument value to validate validation_rules: Dictionary of validation rules param_name: Parameter name for error messages - + Returns: Validated value - + Raises: BadParameter: If validation fails """ if not value and validation_rules.get('required', True): raise BadParameter(f"Required argument {param_name} is empty") - + if 'min_length' in validation_rules and len(value) < validation_rules['min_length']: raise BadParameter( f"Argument {param_name} too short (min {validation_rules['min_length']} characters)" ) - + if 'max_length' in validation_rules and len(value) > validation_rules['max_length']: raise BadParameter( f"Argument {param_name} too long (max {validation_rules['max_length']} characters)" ) - + if 'pattern' in validation_rules: pattern = validation_rules['pattern'] if isinstance(pattern, str): @@ -191,11 +193,11 @@ def validate_argument( raise BadParameter( f"Argument {param_name} doesn't match pattern: {validation_rules.get('pattern', pattern.pattern)}" ) - + if 'choices' in validation_rules: - validate_choice(value, validation_rules['choices'], param_name, + validate_choice(value, validation_rules['choices'], param_name, validation_rules.get('case_sensitive', True)) - + return value def check_mutually_exclusive( @@ -204,11 +206,11 @@ def check_mutually_exclusive( ) -> None: """ Check that mutually exclusive options are not used together. - + Args: params: Dictionary of parsed parameters exclusive_groups: List of option groups that are mutually exclusive - + Raises: UsageError: If mutually exclusive options are used together """ @@ -227,31 +229,31 @@ def load_config_file( ) -> Dict[str, Any]: """ Load configuration from file. - + Args: path: Path to config file format: File format (json, yaml, or auto) required: Whether file is required - + Returns: Configuration dictionary - + Raises: UsageError: If required file not found or invalid format """ path = Path(os.path.expanduser(path)) - + if not path.exists(): if required: raise UsageError(f"Config file not found: {path}") return {} - + # Auto-detect format from extension if format == 'auto': format = path.suffix.lstrip('.').lower() if format not in ('json', 'yaml', 'yml'): raise UsageError(f"Unsupported config format: {format}") - + try: with open(path) as f: if format == 'json': @@ -269,14 +271,14 @@ def parse_key_value( ) -> tuple: """ Parse key-value string. - + Args: value: String in format "key=value" separator: Key-value separator - + Returns: Tuple of (key, value) - + Raises: BadParameter: If string not in key=value format """ @@ -294,11 +296,11 @@ def parse_list( ) -> List[str]: """ Parse comma-separated list. - + Args: value: Comma-separated string separator: List item separator - + Returns: List of strings """ @@ -311,15 +313,15 @@ def parse_dict( ) -> Dict[str, str]: """ Parse dictionary string. - + Args: value: String in format "key1=value1,key2=value2" item_separator: Separator between items key_value_separator: Separator between keys and values - + Returns: Dictionary of key-value pairs - + Example: >>> parse_dict("name=test,count=42") {'name': 'test', 'count': '42'} @@ -327,12 +329,12 @@ def parse_dict( result = {} if not value: return result - + items = parse_list(value, item_separator) for item in items: key, value = parse_key_value(item, key_value_separator) result[key] = value - + return result def get_env_var( @@ -343,24 +345,24 @@ def get_env_var( ) -> Any: """ Get and validate environment variable. - + Args: name: Environment variable name type_: Expected type required: Whether variable is required default: Default value if not set - + Returns: Environment variable value - + Raises: UsageError: If required variable not set """ value = os.environ.get(name) - + if value is None: if required: raise UsageError(f"Required environment variable not set: {name}") return default - + return convert_type(value, type_, name) diff --git a/webscout/update_checker.py b/webscout/update_checker.py index 0d271e57..21c3b804 100644 --- a/webscout/update_checker.py +++ b/webscout/update_checker.py @@ -1,10 +1,11 @@ import importlib.metadata import os import sys -import time import tempfile +import time from pathlib import Path -from typing import Optional, Dict, Any, Literal +from typing import Any, Dict, Optional + import requests from packaging import version @@ -61,20 +62,20 @@ def get_pypi_versions() -> Dict[str, Optional[str]]: response = session.get(PYPI_URL, timeout=3) # Faster timeout response.raise_for_status() data = response.json() - + stable = data.get('info', {}).get('version') - + releases = data.get('releases', {}).keys() parsed_versions = [] for v in releases: try: parsed_versions.append(version.parse(v)) - except: + except Exception: continue - + all_versions = sorted(parsed_versions) latest = str(all_versions[-1]) if all_versions else stable - + return {"stable": stable, "latest": latest} except Exception: return {"stable": None, "latest": None} @@ -83,27 +84,27 @@ def should_check(force: bool = False) -> bool: """Check if we should perform an update check based on cache.""" if os.environ.get("WEBSCOUT_NO_UPDATE"): return False - + if force: return True - + try: if not CACHE_FILE.exists(): return True - + last_check = float(CACHE_FILE.read_text().strip()) # Check every 12 hours if time.time() - last_check > 43200: return True - except: + except Exception: return True return False -def mark_checked(): +def mark_checked() -> Any: """Mark the current time as the last update check.""" try: CACHE_FILE.write_text(str(time.time())) - except: + except Exception: pass def is_venv() -> bool: @@ -124,12 +125,12 @@ def format_update_message(current: str, new: str, utype: str) -> str: cmd = "pip install -U webscout" if utype == "Pre-release" or version.parse(new).is_prerelease: cmd = "pip install -U --pre webscout" - + if HAS_RICH: from io import StringIO capture = StringIO() console = Console(file=capture, force_terminal=True, width=80) - + content = Text.assemble( ("A new ", "white"), (f"{utype} ", "bold yellow" if utype == "Pre-release" else "bold green"), @@ -137,13 +138,13 @@ def format_update_message(current: str, new: str, utype: str) -> str: ("Current: ", "white"), (f"{current}", "bold red"), ("\n", ""), ("Latest: ", "white"), (f"{new}", "bold green"), ("\n\n", ""), ("To update, run: ", "white"), (f"{cmd}", "bold cyan"), ("\n\n", ""), - (f"Subscribe to my YouTube: ", "dim"), (f"{YOUTUBE_URL}", "dim cyan"), ("\n", ""), - (f"Star on GitHub: ", "dim"), (f"{GITHUB_URL}", "dim cyan") + ("Subscribe to my YouTube: ", "dim"), (f"{YOUTUBE_URL}", "dim cyan"), ("\n", ""), + ("Star on GitHub: ", "dim"), (f"{GITHUB_URL}", "dim cyan") ) - + panel = Panel( content, - title=f"[bold magenta]Update Available[/bold magenta]", + title="[bold magenta]Update Available[/bold magenta]", border_style="bright_blue", expand=False, padding=(1, 2) @@ -164,15 +165,15 @@ def format_dev_message(current: str, latest: str) -> str: from io import StringIO capture = StringIO() console = Console(file=capture, force_terminal=True, width=80) - + content = Text.assemble( ("You are running a ", "white"), ("Development Version", "bold yellow"), ("\n\nLocal Version: ", "white"), (f"{current}", "bold cyan"), ("\n", ""), ("Latest PyPI: ", "white"), (f"{latest}", "bold green"), ("\n\n", ""), - (f"YouTube: ", "dim"), (f"{YOUTUBE_URL}", "dim cyan") + ("YouTube: ", "dim"), (f"{YOUTUBE_URL}", "dim cyan") ) - + panel = Panel( content, title="[bold blue]Webscout Dev Mode[/bold blue]", @@ -193,10 +194,10 @@ def format_dev_message(current: str, latest: str) -> str: def check_for_updates(force: bool = False) -> Optional[str]: """ Check if a newer version of Webscout is available. - + Args: force (bool): If True, ignore cache and force check. - + Returns: Optional[str]: Formatted update message or None. """ @@ -210,22 +211,22 @@ def check_for_updates(force: bool = False) -> Optional[str]: try: installed_str = get_installed_version() installed_v = version.parse(installed_str) - + pypi = get_pypi_versions() mark_checked() # Mark even if it fails or no update, to avoid constant hitting - + if not pypi['stable']: return None - + latest_stable_str = pypi['stable'] latest_stable_v = version.parse(latest_stable_str) - + latest_any_str = pypi['latest'] latest_any_v = version.parse(latest_any_str) - + # Decide what to recommend is_prerelease = installed_v.is_prerelease - + if is_prerelease: # User is on pre-release, they should know about ANY newer version if installed_v < latest_any_v: @@ -244,7 +245,7 @@ def check_for_updates(force: bool = False) -> Optional[str]: except Exception: pass # Be silent on errors during auto-check - + return None if __name__ == "__main__": @@ -252,4 +253,4 @@ def check_for_updates(force: bool = False) -> Optional[str]: if msg: print(msg) else: - print("Webscout is up to date!") \ No newline at end of file + print("Webscout is up to date!") diff --git a/webscout/utils.py b/webscout/utils.py index ce70bb9d..2fa3900e 100644 --- a/webscout/utils.py +++ b/webscout/utils.py @@ -84,4 +84,4 @@ def _calculate_distance(lat1: Decimal, lon1: Decimal, lat2: Decimal, lon2: Decim dlon, dlat = rlon2 - rlon1, rlat2 - rlat1 a = sin(dlat / 2) ** 2 + cos(rlat1) * cos(rlat2) * sin(dlon / 2) ** 2 c = 2 * atan2(sqrt(a), sqrt(1 - a)) - return R * c \ No newline at end of file + return R * c diff --git a/webscout/version.py b/webscout/version.py index 3f42086e..a54b8d10 100644 --- a/webscout/version.py +++ b/webscout/version.py @@ -1,2 +1,2 @@ -__version__ = "2025.12.19" +__version__ = "2025.12.20" __prog__ = "webscout" diff --git a/webscout/zeroart/__init__.py b/webscout/zeroart/__init__.py index 28b269ce..206f5543 100644 --- a/webscout/zeroart/__init__.py +++ b/webscout/zeroart/__init__.py @@ -5,16 +5,27 @@ """ from typing import Dict, List, Literal, Optional, Union + from .base import ZeroArtFont -from .fonts import BlockFont, SlantFont, NeonFont, CyberFont, DottedFont, ShadowFont, ThreeDFont, ElectronicFont, IsometricFont from .effects import AsciiArtEffects +from .fonts import ( + BlockFont, + CyberFont, + DottedFont, + ElectronicFont, + IsometricFont, + NeonFont, + ShadowFont, + SlantFont, + ThreeDFont, +) FontType = Literal['block', 'slant', 'neon', 'cyber', 'dotted', 'shadow', '3d', 'electronic', 'isometric'] def figlet_format(text: str, font: Union[str, ZeroArtFont] = 'block') -> str: """ Generate ASCII art text - + :param text: Text to convert :param font: Font style (default: 'block') :return: ASCII art representation of text @@ -30,7 +41,7 @@ def figlet_format(text: str, font: Union[str, ZeroArtFont] = 'block') -> str: 'electronic': ElectronicFont(), 'isometric': IsometricFont() } - + if isinstance(font, str): selected_font: ZeroArtFont = font_map.get(font.lower(), BlockFont()) else: @@ -40,7 +51,7 @@ def figlet_format(text: str, font: Union[str, ZeroArtFont] = 'block') -> str: def print_figlet(text: str, font: Union[str, ZeroArtFont] = 'block') -> None: """ Print ASCII art text directly - + :param text: Text to convert and print :param font: Font style (default: 'block') """ @@ -50,7 +61,7 @@ def print_figlet(text: str, font: Union[str, ZeroArtFont] = 'block') -> None: def rainbow(text: str, font: Union[str, ZeroArtFont] = 'block') -> str: """ Apply a rainbow-like color effect to ASCII art - + :param text: Text to render :param font: Font style (default: 'block') :return: Rainbow-styled ASCII art @@ -66,7 +77,7 @@ def rainbow(text: str, font: Union[str, ZeroArtFont] = 'block') -> str: 'electronic': ElectronicFont(), 'isometric': IsometricFont() } - + if isinstance(font, str): selected_font: ZeroArtFont = font_map.get(font.lower(), BlockFont()) else: @@ -76,7 +87,7 @@ def rainbow(text: str, font: Union[str, ZeroArtFont] = 'block') -> str: def glitch(text: str, font: Union[str, ZeroArtFont] = 'block', glitch_intensity: float = 0.1) -> str: """ Apply a glitch-like distortion to ASCII art - + :param text: Text to render :param font: Font style (default: 'block') :param glitch_intensity: Probability of character distortion @@ -93,7 +104,7 @@ def glitch(text: str, font: Union[str, ZeroArtFont] = 'block', glitch_intensity: 'electronic': ElectronicFont(), 'isometric': IsometricFont() } - + if isinstance(font, str): selected_font: ZeroArtFont = font_map.get(font.lower(), BlockFont()) else: @@ -105,7 +116,7 @@ def glitch(text: str, font: Union[str, ZeroArtFont] = 'block', glitch_intensity: def outline(text: str, font: Union[str, ZeroArtFont] = 'block', outline_char: str = '*') -> str: """ Add an outline effect to ASCII art - + :param text: Text to render :param font: Font style (default: 'block') :param outline_char: Character to use for outline @@ -122,7 +133,7 @@ def outline(text: str, font: Union[str, ZeroArtFont] = 'block', outline_char: st 'electronic': ElectronicFont(), 'isometric': IsometricFont() } - + if isinstance(font, str): selected_font: ZeroArtFont = font_map.get(font.lower(), BlockFont()) else: @@ -132,7 +143,7 @@ def outline(text: str, font: Union[str, ZeroArtFont] = 'block', outline_char: st def gradient(text: str, font: Union[str, ZeroArtFont] = 'block', color1: tuple = (255, 0, 0), color2: tuple = (0, 0, 255)) -> str: """ Apply a gradient color effect to ASCII art - + :param text: Text to render :param font: Font style (default: 'block') :param color1: Starting RGB color @@ -150,7 +161,7 @@ def gradient(text: str, font: Union[str, ZeroArtFont] = 'block', color1: tuple = 'electronic': ElectronicFont(), 'isometric': IsometricFont() } - + if isinstance(font, str): selected_font: ZeroArtFont = font_map.get(font.lower(), BlockFont()) else: @@ -160,7 +171,7 @@ def gradient(text: str, font: Union[str, ZeroArtFont] = 'block', color1: tuple = def bounce(text: str, font: Union[str, ZeroArtFont] = 'block', bounce_height: int = 2) -> str: """ Create a bouncing text effect - + :param text: Text to render :param font: Font style (default: 'block') :param bounce_height: Height of the bounce @@ -177,7 +188,7 @@ def bounce(text: str, font: Union[str, ZeroArtFont] = 'block', bounce_height: in 'electronic': ElectronicFont(), 'isometric': IsometricFont() } - + if isinstance(font, str): selected_font: ZeroArtFont = font_map.get(font.lower(), BlockFont()) else: @@ -185,17 +196,17 @@ def bounce(text: str, font: Union[str, ZeroArtFont] = 'block', bounce_height: in return AsciiArtEffects.bouncing_effect(text, selected_font, bounce_height) __all__ = [ - 'figlet_format', - 'print_figlet', - 'rainbow', - 'glitch', - 'wrap_text', + 'figlet_format', + 'print_figlet', + 'rainbow', + 'glitch', + 'wrap_text', 'outline', 'gradient', 'bounce', - 'BlockFont', - 'SlantFont', - 'NeonFont', + 'BlockFont', + 'SlantFont', + 'NeonFont', 'CyberFont', 'DottedFont', 'ShadowFont', @@ -204,4 +215,4 @@ def bounce(text: str, font: Union[str, ZeroArtFont] = 'block', bounce_height: in 'IsometricFont', 'ZeroArtFont', 'FontType' -] \ No newline at end of file +] diff --git a/webscout/zeroart/base.py b/webscout/zeroart/base.py index dbe9c4c0..35605838 100644 --- a/webscout/zeroart/base.py +++ b/webscout/zeroart/base.py @@ -1,7 +1,8 @@ """ ZeroArt Base: Core classes and utilities for ASCII art generation """ -from typing import Dict, List, Optional, Union +from typing import Dict, List + class ZeroArtFont: """Base class for ASCII art fonts""" @@ -13,7 +14,7 @@ def __init__(self, name: str) -> None: def add_letter(self, char: str, art_lines: List[str]) -> None: """ Add a custom letter to the font - + :param char: Character to add :param art_lines: List of art lines representing the character """ @@ -22,7 +23,7 @@ def add_letter(self, char: str, art_lines: List[str]) -> None: def add_special_char(self, name: str, art_lines: List[str]) -> None: """ Add a special ASCII art character or design - + :param name: Name of the special character :param art_lines: List of art lines representing the character """ @@ -31,7 +32,7 @@ def add_special_char(self, name: str, art_lines: List[str]) -> None: def get_letter(self, char: str) -> List[str]: """ Get ASCII art for a specific character - + :param char: Character to retrieve :return: List of art lines or default space """ @@ -40,7 +41,7 @@ def get_letter(self, char: str) -> List[str]: def render(self, text: str) -> str: """ Render text as ASCII art - + :param text: Text to render as ASCII art :return: ASCII art representation of the text """ @@ -48,19 +49,19 @@ def render(self, text: str) -> str: return "" # Get the maximum height of any character in the font max_height: int = max(len(self.get_letter(c)) for c in text) - + # Initialize art_lines with empty strings art_lines: List[str] = ["" for _ in range(max_height)] - + # Process each character for char in text: char_art: List[str] = self.get_letter(char) # Pad shorter characters with empty lines to match max_height while len(char_art) < max_height: char_art.append(" " * len(char_art[0])) - + # Add character art to each line for i in range(max_height): art_lines[i] += char_art[i] + " " - + return "\n".join(art_lines) diff --git a/webscout/zeroart/effects.py b/webscout/zeroart/effects.py index 94c36c27..cc30cb40 100644 --- a/webscout/zeroart/effects.py +++ b/webscout/zeroart/effects.py @@ -4,17 +4,19 @@ import random import textwrap -from typing import List, Optional, Union +from typing import List + from .base import ZeroArtFont - + + class AsciiArtEffects: """Collection of ASCII art text effects""" - + @staticmethod def rainbow_effect(text: str, font: ZeroArtFont) -> str: """ Apply a rainbow-like color effect to ASCII art - + :param text: Text to render :param font: Font to use :return: Rainbow-styled ASCII art @@ -26,10 +28,10 @@ def rainbow_effect(text: str, font: ZeroArtFont) -> str: '\033[94m', # Blue '\033[95m', # Magenta ] - + art: str = font.render(text) art_lines: List[str] = art.split('\n') - + colored_lines: List[str] = [] for line in art_lines: colored_line: str = '' @@ -37,14 +39,14 @@ def rainbow_effect(text: str, font: ZeroArtFont) -> str: color: str = random.choice(colors) colored_line += color + char colored_lines.append(colored_line + '\033[0m') # Reset color - + return '\n'.join(colored_lines) - + @staticmethod def glitch_effect(text: str, font: ZeroArtFont, glitch_intensity: float = 0.1) -> str: """ Apply a glitch-like distortion to ASCII art - + :param text: Text to render :param font: Font to use :param glitch_intensity: Probability of character distortion @@ -52,10 +54,10 @@ def glitch_effect(text: str, font: ZeroArtFont, glitch_intensity: float = 0.1) - """ art: str = font.render(text) art_lines: List[str] = art.split('\n') - + glitched_lines: List[str] = [] glitch_chars: List[str] = ['~', '^', '`', '¯', '±'] - + for line in art_lines: glitched_line: str = '' for char in line: @@ -64,14 +66,14 @@ def glitch_effect(text: str, font: ZeroArtFont, glitch_intensity: float = 0.1) - else: glitched_line += char glitched_lines.append(glitched_line) - + return '\n'.join(glitched_lines) @staticmethod def wrap_text(text: str, width: int = 20) -> str: """ Wrap ASCII art text to a specific width - + :param text: Text to wrap :param width: Maximum line width :return: Wrapped text @@ -82,7 +84,7 @@ def wrap_text(text: str, width: int = 20) -> str: def outline_effect(text: str, font: ZeroArtFont, outline_char: str = '*') -> str: """ Add an outline effect to ASCII art - + :param text: Text to render :param font: Font to use :param outline_char: Character to use for outline @@ -90,21 +92,21 @@ def outline_effect(text: str, font: ZeroArtFont, outline_char: str = '*') -> str """ art: str = font.render(text) art_lines: List[str] = art.split('\n') - + outlined_lines: List[str] = [] for line in art_lines: outlined_line: str = outline_char + line + outline_char outlined_lines.append(outlined_line) - + top_bottom_line: str = outline_char * (len(outlined_lines[0])) - + return '\n'.join([top_bottom_line] + outlined_lines + [top_bottom_line]) @staticmethod def gradient_effect(text: str, font: ZeroArtFont, color1: tuple = (255, 0, 0), color2: tuple = (0, 0, 255)) -> str: """ Apply a gradient color effect to ASCII art - + :param text: Text to render :param font: Font to use :param color1: Starting RGB color @@ -113,28 +115,28 @@ def gradient_effect(text: str, font: ZeroArtFont, color1: tuple = (255, 0, 0), c """ art: str = font.render(text) art_lines: List[str] = art.split('\n') - + gradient_lines: List[str] = [] num_lines = len(art_lines) - + for i, line in enumerate(art_lines): # Calculate interpolated color ratio = i / max(1, num_lines - 1) r = int(color1[0] * (1 - ratio) + color2[0] * ratio) g = int(color1[1] * (1 - ratio) + color2[1] * ratio) b = int(color1[2] * (1 - ratio) + color2[2] * ratio) - + # Apply ANSI color colored_line = f'\033[38;2;{r};{g};{b}m{line}\033[0m' gradient_lines.append(colored_line) - + return '\n'.join(gradient_lines) @staticmethod def bouncing_effect(text: str, font: ZeroArtFont, bounce_height: int = 2) -> str: """ Create a bouncing text effect - + :param text: Text to render :param font: Font to use :param bounce_height: Height of the bounce @@ -142,10 +144,10 @@ def bouncing_effect(text: str, font: ZeroArtFont, bounce_height: int = 2) -> str """ art: str = font.render(text) art_lines: List[str] = art.split('\n') - + bouncing_lines: List[str] = [] for i, line in enumerate(art_lines): offset = abs(bounce_height - i % (2 * bounce_height)) bouncing_lines.append(" " * offset + line) - - return '\n'.join(bouncing_lines) \ No newline at end of file + + return '\n'.join(bouncing_lines) diff --git a/webscout/zeroart/fonts.py b/webscout/zeroart/fonts.py index 6c841d4a..b9f367ea 100644 --- a/webscout/zeroart/fonts.py +++ b/webscout/zeroart/fonts.py @@ -1,10 +1,12 @@ """ ZeroArt Fonts: Predefined ASCII art fonts -""" +""" from typing import Dict, List + from .base import ZeroArtFont + class BlockFont(ZeroArtFont): """Block-style ASCII art font""" def __init__(self) -> None: @@ -200,7 +202,7 @@ def _populate_letters(self) -> None: " ███████ " ] } - + self.letters.update(block_letters) class SlantFont(ZeroArtFont): @@ -416,7 +418,7 @@ def _populate_letters(self) -> None: "/____|_| " ] } - + self.letters.update(slant_letters) # Add more custom fonts here @@ -615,7 +617,7 @@ def _populate_letters(self) -> None: " ████████ " ] } - + self.letters.update(neon_letters) class CyberFont(ZeroArtFont): @@ -813,7 +815,7 @@ def _populate_letters(self) -> None: " ▓▓▓▓▓▓▓ " ] } - + self.letters.update(cyber_letters) class DottedFont(ZeroArtFont): @@ -1011,7 +1013,7 @@ def _populate_letters(self) -> None: " :::::::: " ] } - + self.letters.update(dotted_letters) class ShadowFont(ZeroArtFont): @@ -1235,7 +1237,7 @@ def _populate_letters(self) -> None: "▓▓▓▓▓▓▓▓ " ] } - + self.letters.update(shadow_letters) class ThreeDFont(ZeroArtFont): @@ -1451,7 +1453,7 @@ def _populate_letters(self) -> None: "/____|" ] } - + self.letters.update(three_d_letters) class ElectronicFont(ZeroArtFont): @@ -1623,7 +1625,7 @@ def _populate_letters(self) -> None: "===" ] } - + self.letters.update(electronic_letters) class IsometricFont(ZeroArtFont): @@ -1793,5 +1795,5 @@ def _populate_letters(self) -> None: "----" ] } - - self.letters.update(isometric_letters) \ No newline at end of file + + self.letters.update(isometric_letters)