refactor: packages folder be 2 (#1628)

# Description

Move some folders to packages
Remove unused code

backend/celery_task.py

@@ -1,8 +1,8 @@
 from celery import shared_task
 from models.brains import Brain
 from models.settings import get_supabase_db
+from packages.embeddings.vectors import Neurons
 from repository.files.upload_file import DocumentSerializable
-from utils.vectors import Neurons
 
 
 @shared_task

backend/celery_worker.py

@@ -4,12 +4,12 @@
 
 from celery import Celery
 from celery.schedules import crontab
-from crawl.crawler import CrawlWebsite
 from fastapi import UploadFile
 from models.databases.supabase.notifications import NotificationUpdatableProperties
 from models.files import File
 from models.notifications import NotificationsStatusEnum
 from models.settings import get_supabase_client
+from packages.files.crawl.crawler import CrawlWebsite
 from packages.files.parsers.github import process_github
 from packages.files.processors import filter_file
 from repository.brain.update_brain_last_update_time import update_brain_last_update_time

backend/chat_service.py

@@ -1,6 +1,8 @@
 import os
 
-from utils import handle_request_validation_error
+from packages.utils.handle_request_validation_error import (
+    handle_request_validation_error,
+)
 
 if __name__ == "__main__":
     # import needed here when running main.py to debug backend

backend/crawl_service.py

@@ -1,6 +1,6 @@
 import os
 
-from utils import handle_request_validation_error
+from packages.utils import handle_request_validation_error
 
 if __name__ == "__main__":
     # import needed here when running main.py to debug backend

backend/main.py

@@ -1,6 +1,6 @@
 import os
 
-from utils import handle_request_validation_error
+from packages.utils import handle_request_validation_error
 
 if __name__ == "__main__":
     # import needed here when running main.py to debug backend
@@ -17,6 +17,7 @@
 from routes.api_key_routes import api_key_router
 from routes.brain_routes import brain_router
 from routes.chat_routes import chat_router
+from routes.contact_routes import router as contact_router
 from routes.crawl_routes import crawl_router
 from routes.explore_routes import explore_router
 from routes.knowledge_routes import knowledge_router
@@ -27,7 +28,6 @@
 from routes.subscription_routes import subscription_router
 from routes.upload_routes import upload_router
 from routes.user_routes import user_router
-from routes.contact_routes import router as contact_router
 
 logger = get_logger(__name__)
 

backend/auth/auth_bearer.py → backend/middlewares/auth/auth_bearer.py

@@ -1,10 +1,10 @@
 import os
 from typing import Optional
 
-from auth.api_key_handler import get_user_from_api_key, verify_api_key
-from auth.jwt_token_handler import decode_access_token, verify_token
 from fastapi import Depends, HTTPException, Request
 from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from middlewares.auth.api_key_handler import get_user_from_api_key, verify_api_key
+from middlewares.auth.jwt_token_handler import decode_access_token, verify_token
 from models import UserIdentity
 
 

backend/models/brains.py

@@ -2,12 +2,11 @@
 from uuid import UUID
 
 from logger import get_logger
-from pydantic import BaseModel
-from supabase.client import Client
-from utils.vectors import get_unique_files_from_vector_ids
-
 from models.databases.supabase.supabase import SupabaseDB
 from models.settings import get_supabase_client, get_supabase_db
+from packages.embeddings.vectors import get_unique_files_from_vector_ids
+from pydantic import BaseModel
+from supabase.client import Client
 
 logger = get_logger(__name__)
 

backend/models/databases/supabase/vectors.py

@@ -40,6 +40,7 @@ def set_file_sha_from_metadata(self, file_sha1):
 
         return response
 
+    # TODO: remove duplicate similarity_search in supabase vector store
     def similarity_search(self, query_embedding, table, top_k, threshold):
         response = self.db.rpc(
             table,

backend/utils/send_email.py → backend/packages/emails/send_email.py

@@ -1,7 +1,9 @@
-import resend
 from typing import Dict
+
+import resend
 from models import ResendSettings
 
+
 def send_email(params: Dict):
     settings = ResendSettings()
     resend.api_key = settings.resend_api_key

backend/utils/vectors.py → backend/packages/embeddings/vectors.py

@@ -10,6 +10,7 @@
 logger = get_logger(__name__)
 
 
+# TODO: Create interface for embeddings and implement it for Supabase and OpenAI (current Quivr)
 class Neurons(BaseModel):
     def create_vector(self, doc, user_openai_api_key=None):
         documents_vector_store = get_documents_vector_store()
@@ -31,14 +32,6 @@ def create_embedding(self, content):
         embeddings = get_embeddings()
         return embeddings.embed_query(content)
 
-    def similarity_search(self, query, table="match_summaries", top_k=6, threshold=0.5):
-        query_embedding = self.create_embedding(query)
-        supabase_db = get_supabase_db()
-        summaries = supabase_db.similarity_search(
-            query_embedding, table, top_k, threshold
-        )
-        return summaries.data
-
 
 def error_callback(exception):
     print("An exception occurred:", exception)
@@ -56,6 +49,7 @@ def process_batch(batch_ids: List[str]):
         logger.error("Error retrieving batched vectors", e)
 
 
+# TODO: move to Knowledge class
 def get_unique_files_from_vector_ids(vectors_ids: List[str]):
     # Move into Vectors class
     """

backend/packages/files/parsers/github.py

@@ -5,8 +5,8 @@
 from langchain.schema import Document
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from models import Brain, File
+from packages.embeddings.vectors import Neurons
 from packages.files.file import compute_sha1_from_content
-from utils.vectors import Neurons
 
 
 async def process_github(

backend/packages/files/parsers/telegram.py

@@ -1,5 +1,5 @@
-from loaders.telegram import TelegramChatFileLoader
 from models import File
+from packages.files.loaders.telegram import TelegramChatFileLoader
 
 from .common import process_file
 

backend/repository/brain_subscription/resend_invitation_email.py

@@ -1,9 +1,8 @@
 from logger import get_logger
-from models import BrainSubscription, BrainSettings
-
+from models import BrainSettings, BrainSubscription
+from packages.emails.send_email import send_email
 from repository.brain import get_brain_details
 from repository.brain_subscription import get_brain_url
-from utils.send_email import send_email
 
 logger = get_logger(__name__)
 

backend/repository/chat/get_chat_history_with_notifications.py

@@ -3,15 +3,10 @@
 from uuid import UUID
 
 from models.notifications import Notification
+from packages.utils import parse_message_time
 from pydantic import BaseModel
-from utils.parse_message_time import (
-    parse_message_time,
-)
-
 from repository.chat.get_chat_history import GetChatHistoryOutput, get_chat_history
-from repository.notification.get_chat_notifications import (
-    get_chat_notifications,
-)
+from repository.notification.get_chat_notifications import get_chat_notifications
 
 
 class ChatItemType(Enum):

backend/routes/api_key_routes.py

@@ -3,9 +3,9 @@
 from uuid import uuid4
 
 from asyncpg.exceptions import UniqueViolationError
-from auth import AuthBearer, get_current_user
 from fastapi import APIRouter, Depends
 from logger import get_logger
+from middlewares.auth import AuthBearer, get_current_user
 from models import UserIdentity, get_supabase_db
 from pydantic import BaseModel
 

backend/routes/authorizations/brain_authorization.py

@@ -1,12 +1,11 @@
 from typing import List, Optional, Union
 from uuid import UUID
 
-from auth.auth_bearer import get_current_user
 from fastapi import Depends, HTTPException, status
+from middlewares.auth.auth_bearer import get_current_user
 from models import UserIdentity
 from repository.brain import get_brain_for_user
 from repository.brain.get_brain_details import get_brain_details
-
 from routes.authorizations.types import RoleEnum
 
 

backend/routes/brain_routes.py

@@ -1,8 +1,8 @@
 from uuid import UUID
 
-from auth import AuthBearer, get_current_user
 from fastapi import APIRouter, Depends, HTTPException
 from logger import get_logger
+from middlewares.auth.auth_bearer import AuthBearer, get_current_user
 from models import UserIdentity, UserUsage
 from models.brain_entity import PublicBrain
 from models.databases.supabase.brains import (
@@ -24,7 +24,6 @@
     update_brain_by_id,
 )
 from repository.prompt import delete_prompt_by_id, get_prompt_by_id
-
 from routes.authorizations.brain_authorization import has_brain_authorization
 from routes.authorizations.types import RoleEnum
 

backend/routes/chat_routes.py

@@ -2,11 +2,11 @@
 from uuid import UUID
 from venv import logger
 
-from auth import AuthBearer, get_current_user
 from fastapi import APIRouter, Depends, HTTPException, Query, Request
 from fastapi.responses import StreamingResponse
 from llm.qa_base import QABaseBrainPicking
 from llm.qa_headless import HeadlessQA
+from middlewares.auth import AuthBearer, get_current_user
 from models import (
     Brain,
     BrainEntity,
@@ -32,7 +32,6 @@
     get_chat_history_with_notifications,
 )
 from repository.notification.remove_chat_notifications import remove_chat_notifications
-
 from routes.chat.factory import get_chat_strategy
 from routes.chat.utils import (
     NullableUUID,

backend/routes/contact_routes.py

@@ -1,10 +1,9 @@
 from fastapi import APIRouter
 from logger import get_logger
 from models import ContactsSettings
+from packages.emails.send_email import send_email
 from pydantic import BaseModel
 
-from utils.send_email import send_email
-
 
 class ContactMessage(BaseModel):
     customer_email: str

backend/routes/crawl_routes.py

@@ -1,15 +1,15 @@
 from typing import Optional
 from uuid import UUID
 
-from auth import AuthBearer, get_current_user
 from celery_worker import process_crawl_and_notify
-from crawl.crawler import CrawlWebsite
 from fastapi import APIRouter, Depends, Query, Request
 from logger import get_logger
+from middlewares.auth import AuthBearer, get_current_user
 from models import Brain, UserIdentity, UserUsage
 from models.databases.supabase.knowledge import CreateKnowledgeProperties
 from models.databases.supabase.notifications import CreateNotificationProperties
 from models.notifications import NotificationsStatusEnum
+from packages.files.crawl.crawler import CrawlWebsite
 from packages.files.file import convert_bytes
 from repository.knowledge.add_knowledge import add_knowledge
 from repository.notification.add_notification import add_notification

backend/routes/explore_routes.py

@@ -1,7 +1,7 @@
 from uuid import UUID
 
-from auth import AuthBearer, get_current_user
 from fastapi import APIRouter, Depends, Query
+from middlewares.auth import AuthBearer, get_current_user
 from models import Brain, UserIdentity, get_supabase_db
 from routes.authorizations.brain_authorization import (
     RoleEnum,

backend/routes/knowledge_routes.py

@@ -1,15 +1,14 @@
 from uuid import UUID
 
-from auth import AuthBearer, get_current_user
 from fastapi import APIRouter, Depends, Query
 from logger import get_logger
+from middlewares.auth import AuthBearer, get_current_user
 from models import Brain, UserIdentity
 from repository.files.delete_file import delete_file_from_storage
 from repository.files.generate_file_signed_url import generate_file_signed_url
 from repository.knowledge.get_all_knowledge import get_all_knowledge
 from repository.knowledge.get_knowledge import get_knowledge
 from repository.knowledge.remove_knowledge import remove_knowledge
-
 from routes.authorizations.brain_authorization import (
     RoleEnum,
     has_brain_authorization,

backend/routes/notification_routes.py

@@ -1,10 +1,8 @@
 from uuid import UUID
 
-from auth import AuthBearer
 from fastapi import APIRouter, Depends
-from repository.notification.get_chat_notifications import (
-    get_chat_notifications,
-)
+from middlewares.auth import AuthBearer
+from repository.notification.get_chat_notifications import get_chat_notifications
 
 notification_router = APIRouter()
 

backend/routes/onboarding_routes.py

@@ -1,8 +1,8 @@
-from auth import (
-    AuthBearer,
-    get_current_user,  # Assuming you have a get_current_user function
-)
 from fastapi import APIRouter, Depends
+from middlewares.auth import (
+    get_current_user,
+)  # Assuming you have a get_current_user function
+from middlewares.auth import AuthBearer
 from models.databases.supabase.onboarding import (
     OnboardingStates,
     OnboardingUpdatableProperties,

backend/routes/prompt_routes.py

@@ -1,12 +1,12 @@
 from uuid import UUID
 
-from auth import AuthBearer
 from fastapi import APIRouter, Depends
+from middlewares.auth import AuthBearer
+from models import Prompt
 from models.databases.supabase.prompts import (
     CreatePromptProperties,
     PromptUpdatableProperties,
 )
-from models import Prompt
 from repository.prompt import (
     create_prompt,
     get_prompt_by_id,

backend/routes/subscription_routes.py

@@ -1,8 +1,8 @@
 from typing import List
 from uuid import UUID
 
-from auth.auth_bearer import AuthBearer, get_current_user
 from fastapi import APIRouter, Depends, HTTPException
+from middlewares.auth.auth_bearer import AuthBearer, get_current_user
 from models import BrainSubscription, PromptStatusEnum, UserIdentity
 from pydantic import BaseModel
 from repository.brain import (
@@ -21,7 +21,6 @@
 )
 from repository.prompt import delete_prompt_by_id, get_prompt_by_id
 from repository.user import get_user_id_by_user_email
-
 from routes.authorizations.brain_authorization import (
     RoleEnum,
     has_brain_authorization,

backend/routes/upload_routes.py

@@ -2,10 +2,10 @@
 from typing import Optional
 from uuid import UUID
 
-from auth import AuthBearer, get_current_user
 from celery_worker import process_file_and_notify
 from fastapi import APIRouter, Depends, HTTPException, Query, Request, UploadFile
 from logger import get_logger
+from middlewares.auth import AuthBearer, get_current_user
 from models import Brain, UserIdentity, UserUsage
 from models.databases.supabase.knowledge import CreateKnowledgeProperties
 from models.databases.supabase.notifications import CreateNotificationProperties

backend/routes/user_routes.py

@@ -1,7 +1,7 @@
 import time
 
-from auth import AuthBearer, get_current_user
 from fastapi import APIRouter, Depends, Request
+from middlewares.auth import AuthBearer, get_current_user
 from models import Brain, UserIdentity, UserUsage
 from repository.brain import get_user_default_brain
 from repository.user_identity.get_user_identity import get_user_identity

backend/upload_service.py

@@ -1,6 +1,6 @@
 import os
 
-from utils import handle_request_validation_error
+from packages.utils import handle_request_validation_error
 
 if __name__ == "__main__":
     # import needed here when running main.py to debug backend