Merge branch 'Significant-Gravitas:master' into test-agent

autogpt/app.py

@@ -10,6 +10,7 @@
 from autogpt.processing.text import summarize_text
 from autogpt.prompts.generator import PromptGenerator
 from autogpt.speech import say_text
+from autogpt.url_utils.validators import validate_url
 
 CFG = Config()
 AGENT_MANAGER = AgentManager()
@@ -141,6 +142,7 @@ def execute_command(
 @command(
     "get_text_summary", "Get text summary", '"url": "<url>", "question": "<question>"'
 )
+@validate_url
 def get_text_summary(url: str, question: str) -> str:
     """Return the results of a Google search
 
@@ -157,6 +159,7 @@ def get_text_summary(url: str, question: str) -> str:
 
 
 @command("get_hyperlinks", "Get text summary", '"url": "<url>"')
+@validate_url
 def get_hyperlinks(url: str) -> Union[str, List[str]]:
     """Return the results of a Google search
 

autogpt/cli.py

@@ -136,6 +136,26 @@ def main(
                     "Please consider upgrading to Python 3.10 or higher.",
                 )
 
+        # TODO: have this directory live outside the repository (e.g. in a user's
+        #   home directory) and have it come in as a command line argument or part of
+        #   the env file.
+        if workspace_directory is None:
+            workspace_directory = Path(__file__).parent / "auto_gpt_workspace"
+        else:
+            workspace_directory = Path(workspace_directory)
+        # TODO: pass in the ai_settings file and the env file and have them cloned into
+        #   the workspace directory so we can bind them to the agent.
+        workspace_directory = Workspace.make_workspace(workspace_directory)
+        cfg.workspace_path = str(workspace_directory)
+
+        # HACK: doing this here to collect some globals that depend on the workspace.
+        file_logger_path = workspace_directory / "file_logger.txt"
+        if not file_logger_path.exists():
+            with file_logger_path.open(mode="w", encoding="utf-8") as f:
+                f.write("File Operation Logger ")
+
+        cfg.file_logger_path = str(file_logger_path)
+
         cfg.set_plugins(scan_plugins(cfg, cfg.debug_mode))
         # Create a CommandRegistry instance and scan default folder
         command_registry = CommandRegistry()
@@ -175,26 +195,6 @@ def main(
         if cfg.debug_mode:
             logger.typewriter_log("Prompt:", Fore.GREEN, system_prompt)
 
-        # TODO: have this directory live outside the repository (e.g. in a user's
-        #   home directory) and have it come in as a command line argument or part of
-        #   the env file.
-        if workspace_directory is None:
-            workspace_directory = Path(__file__).parent / "auto_gpt_workspace"
-        else:
-            workspace_directory = Path(workspace_directory)
-        # TODO: pass in the ai_settings file and the env file and have them cloned into
-        #   the workspace directory so we can bind them to the agent.
-        workspace_directory = Workspace.make_workspace(workspace_directory)
-        cfg.workspace_path = str(workspace_directory)
-
-        # HACK: doing this here to collect some globals that depend on the workspace.
-        file_logger_path = workspace_directory / "file_logger.txt"
-        if not file_logger_path.exists():
-            with file_logger_path.open(mode="w", encoding="utf-8") as f:
-                f.write("File Operation Logger ")
-
-        cfg.file_logger_path = str(file_logger_path)
-
         agent = Agent(
             ai_name=ai_name,
             memory=memory,

autogpt/commands/git_operations.py

@@ -3,6 +3,7 @@
 
 from autogpt.commands.command import command
 from autogpt.config import Config
+from autogpt.url_utils.validators import validate_url
 
 CFG = Config()
 
@@ -14,6 +15,7 @@
     CFG.github_username and CFG.github_api_key,
     "Configure github_username and github_api_key.",
 )
+@validate_url
 def clone_repository(repository_url: str, clone_path: str) -> str:
     """Clone a GitHub repository locally.
 

autogpt/commands/web_requests.py

@@ -1,89 +1,21 @@
 """Browse a webpage and summarize it using the LLM model"""
 from __future__ import annotations
 
-from urllib.parse import urljoin, urlparse
-
 import requests
 from bs4 import BeautifulSoup
 from requests import Response
-from requests.compat import urljoin
 
 from autogpt.config import Config
-from autogpt.memory import get_memory
 from autogpt.processing.html import extract_hyperlinks, format_hyperlinks
+from autogpt.url_utils.validators import validate_url
 
 CFG = Config()
-memory = get_memory(CFG)
 
 session = requests.Session()
 session.headers.update({"User-Agent": CFG.user_agent})
 
 
-def is_valid_url(url: str) -> bool:
-    """Check if the URL is valid
-
-    Args:
-        url (str): The URL to check
-
-    Returns:
-        bool: True if the URL is valid, False otherwise
-    """
-    try:
-        result = urlparse(url)
-        return all([result.scheme, result.netloc])
-    except ValueError:
-        return False
-
-
-def sanitize_url(url: str) -> str:
-    """Sanitize the URL
-
-    Args:
-        url (str): The URL to sanitize
-
-    Returns:
-        str: The sanitized URL
-    """
-    return urljoin(url, urlparse(url).path)
-
-
-def check_local_file_access(url: str) -> bool:
-    """Check if the URL is a local file
-
-    Args:
-        url (str): The URL to check
-
-    Returns:
-        bool: True if the URL is a local file, False otherwise
-    """
-    local_prefixes = [
-        "file:///",
-        "file://localhost/",
-        "file://localhost",
-        "http://localhost",
-        "http://localhost/",
-        "https://localhost",
-        "https://localhost/",
-        "http://2130706433",
-        "http://2130706433/",
-        "https://2130706433",
-        "https://2130706433/",
-        "http://127.0.0.1/",
-        "http://127.0.0.1",
-        "https://127.0.0.1/",
-        "https://127.0.0.1",
-        "https://0.0.0.0/",
-        "https://0.0.0.0",
-        "http://0.0.0.0/",
-        "http://0.0.0.0",
-        "http://0000",
-        "http://0000/",
-        "https://0000",
-        "https://0000/",
-    ]
-    return any(url.startswith(prefix) for prefix in local_prefixes)
-
-
+@validate_url
 def get_response(
     url: str, timeout: int = 10
 ) -> tuple[None, str] | tuple[Response, None]:
@@ -101,17 +33,7 @@ def get_response(
         requests.exceptions.RequestException: If the HTTP request fails
     """
     try:
-        # Restrict access to local files
-        if check_local_file_access(url):
-            raise ValueError("Access to local files is restricted")
-
-        # Most basic check if the URL is valid:
-        if not url.startswith("http://") and not url.startswith("https://"):
-            raise ValueError("Invalid URL format")
-
-        sanitized_url = sanitize_url(url)
-
-        response = session.get(sanitized_url, timeout=timeout)
+        response = session.get(url, timeout=timeout)
 
         # Check if the response contains an HTTP error
         if response.status_code >= 400:

autogpt/commands/web_selenium.py

@@ -21,6 +21,7 @@
 from autogpt.commands.command import command
 from autogpt.config import Config
 from autogpt.processing.html import extract_hyperlinks, format_hyperlinks
+from autogpt.url_utils.validators import validate_url
 
 FILE_DIR = Path(__file__).parent.parent
 CFG = Config()
@@ -31,6 +32,7 @@
     "Browse Website",
     '"url": "<url>", "question": "<what_you_want_to_find_on_website>"',
 )
+@validate_url
 def browse_website(url: str, question: str) -> tuple[str, WebDriver]:
     """Browse a website and return the answer and links to the user
 

autogpt/llm_utils.py

@@ -5,7 +5,7 @@
 
 import openai
 from colorama import Fore, Style
-from openai.error import APIError, RateLimitError
+from openai.error import APIError, RateLimitError, Timeout
 
 from autogpt.api_manager import api_manager
 from autogpt.config import Config
@@ -123,7 +123,7 @@ def create_chat_completion(
                     + f"You can read more here: {Fore.CYAN}https://github.com/Significant-Gravitas/Auto-GPT#openai-api-keys-configuration{Fore.RESET}"
                 )
                 warned_user = True
-        except APIError as e:
+        except (APIError, Timeout) as e:
             if e.http_status != 502:
                 raise
             if attempt == num_retries - 1:
@@ -154,6 +154,13 @@ def create_chat_completion(
     return resp
 
 
+def get_ada_embedding(text):
+    text = text.replace("\n", " ")
+    return api_manager.embedding_create(
+        text_list=[text], model="text-embedding-ada-002"
+    )
+
+
 def create_embedding_with_ada(text) -> list:
     """Create an embedding with text-ada-002 using the OpenAI SDK"""
     num_retries = 10
@@ -165,7 +172,7 @@ def create_embedding_with_ada(text) -> list:
             )
         except RateLimitError:
             pass
-        except APIError as e:
+        except (APIError, Timeout) as e:
             if e.http_status != 502:
                 raise
             if attempt == num_retries - 1:

autogpt/memory/base.py

@@ -1,21 +1,11 @@
 """Base class for memory providers."""
 import abc
 
-import openai
-
-from autogpt.api_manager import api_manager
 from autogpt.config import AbstractSingleton, Config
 
 cfg = Config()
 
 
-def get_ada_embedding(text):
-    text = text.replace("\n", " ")
-    return api_manager.embedding_create(
-        text_list=[text], model="text-embedding-ada-002"
-    )
-
-
 class MemoryProviderSingleton(AbstractSingleton):
     @abc.abstractmethod
     def add(self, data):

autogpt/memory/local.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import dataclasses
-import os
+from pathlib import Path
 from typing import Any, List
 
 import numpy as np
@@ -38,26 +38,16 @@ def __init__(self, cfg) -> None:
         Returns:
             None
         """
-        self.filename = f"{cfg.memory_index}.json"
-        if os.path.exists(self.filename):
-            try:
-                with open(self.filename, "w+b") as f:
-                    file_content = f.read()
-                    if not file_content.strip():
-                        file_content = b"{}"
-                        f.write(file_content)
-
-                    loaded = orjson.loads(file_content)
-                    self.data = CacheContent(**loaded)
-            except orjson.JSONDecodeError:
-                print(f"Error: The file '{self.filename}' is not in JSON format.")
-                self.data = CacheContent()
-        else:
-            print(
-                f"Warning: The file '{self.filename}' does not exist. "
-                "Local memory would not be saved to a file."
-            )
-            self.data = CacheContent()
+        workspace_path = Path(cfg.workspace_path)
+        self.filename = workspace_path / f"{cfg.memory_index}.json"
+
+        self.filename.touch(exist_ok=True)
+
+        file_content = b"{}"
+        with self.filename.open("w+b") as f:
+            f.write(file_content)
+
+        self.data = CacheContent()
 
     def add(self, text: str):
         """
@@ -92,7 +82,7 @@ def add(self, text: str):
 
     def clear(self) -> str:
         """
-        Clears the redis server.
+        Clears the data in memory.
 
         Returns: A message indicating that the memory has been cleared.
         """

autogpt/memory/milvus.py

@@ -4,7 +4,8 @@
 from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections
 
 from autogpt.config import Config
-from autogpt.memory.base import MemoryProviderSingleton, get_ada_embedding
+from autogpt.llm_utils import get_ada_embedding
+from autogpt.memory.base import MemoryProviderSingleton
 
 
 class MilvusMemory(MemoryProviderSingleton):

autogpt/memory/weaviate.py

@@ -1,12 +1,10 @@
-import uuid
-
 import weaviate
 from weaviate import Client
 from weaviate.embedded import EmbeddedOptions
 from weaviate.util import generate_uuid5
 
-from autogpt.config import Config
-from autogpt.memory.base import MemoryProviderSingleton, get_ada_embedding
+from autogpt.llm_utils import get_ada_embedding
+from autogpt.memory.base import MemoryProviderSingleton
 
 
 def default_schema(weaviate_index):

autogpt/processing/text.py

@@ -10,7 +10,6 @@
 from autogpt.memory import get_memory
 
 CFG = Config()
-MEMORY = get_memory(CFG)
 
 
 def split_text(
@@ -109,7 +108,8 @@ def summarize_text(
 
         memory_to_add = f"Source: {url}\n" f"Raw content part#{i + 1}: {chunk}"
 
-        MEMORY.add(memory_to_add)
+        memory = get_memory(CFG)
+        memory.add(memory_to_add)
 
         messages = [create_message(chunk, question)]
         tokens_for_chunk = token_counter.count_message_tokens(messages, model)
@@ -128,7 +128,7 @@ def summarize_text(
 
         memory_to_add = f"Source: {url}\n" f"Content summary part#{i + 1}: {summary}"
 
-        MEMORY.add(memory_to_add)
+        memory.add(memory_to_add)
 
     print(f"Summarized {len(chunks)} chunks.")