From dceaa5a717bc298be8d6120cef9dbcb473f77ce9 Mon Sep 17 00:00:00 2001 From: gozineb Date: Tue, 14 Nov 2023 11:17:05 +0100 Subject: [PATCH 1/6] =?UTF-8?q?=F0=9F=9A=9A=20move=20email=20utils=20to=20?= =?UTF-8?q?packages?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/{utils => packages/emails}/send_email.py | 4 +++- .../repository/brain_subscription/resend_invitation_email.py | 5 ++--- backend/routes/contact_routes.py | 3 +-- 3 files changed, 6 insertions(+), 6 deletions(-) rename backend/{utils => packages/emails}/send_email.py (99%) diff --git a/backend/utils/send_email.py b/backend/packages/emails/send_email.py similarity index 99% rename from backend/utils/send_email.py rename to backend/packages/emails/send_email.py index 065d1a9818a7..aeba717715a1 100644 --- a/backend/utils/send_email.py +++ b/backend/packages/emails/send_email.py @@ -1,7 +1,9 @@ -import resend from typing import Dict + +import resend from models import ResendSettings + def send_email(params: Dict): settings = ResendSettings() resend.api_key = settings.resend_api_key diff --git a/backend/repository/brain_subscription/resend_invitation_email.py b/backend/repository/brain_subscription/resend_invitation_email.py index 4e409a6dc763..95317eb73ab2 100644 --- a/backend/repository/brain_subscription/resend_invitation_email.py +++ b/backend/repository/brain_subscription/resend_invitation_email.py @@ -1,9 +1,8 @@ from logger import get_logger -from models import BrainSubscription, BrainSettings - +from models import BrainSettings, BrainSubscription +from packages.emails.send_email import send_email from repository.brain import get_brain_details from repository.brain_subscription import get_brain_url -from utils.send_email import send_email logger = get_logger(__name__) diff --git a/backend/routes/contact_routes.py b/backend/routes/contact_routes.py index ae1b677f223f..9911ce72d783 100644 --- a/backend/routes/contact_routes.py +++ b/backend/routes/contact_routes.py @@ -1,10 +1,9 @@ from fastapi import APIRouter from logger import get_logger from models import ContactsSettings +from packages.emails.send_email import send_email from pydantic import BaseModel -from utils.send_email import send_email - class ContactMessage(BaseModel): customer_email: str From 585042a71267d22928ab11cbe03334681fc75b3d Mon Sep 17 00:00:00 2001 From: gozineb Date: Tue, 14 Nov 2023 11:24:14 +0100 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=9A=9A=20embeddings=20package?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/celery_task.py | 2 +- backend/models/brains.py | 7 +++---- backend/{utils => packages/embeddings}/vectors.py | 2 ++ backend/packages/files/parsers/github.py | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) rename backend/{utils => packages/embeddings}/vectors.py (95%) diff --git a/backend/celery_task.py b/backend/celery_task.py index 15d4992e5935..939643996bb9 100644 --- a/backend/celery_task.py +++ b/backend/celery_task.py @@ -1,8 +1,8 @@ from celery import shared_task from models.brains import Brain from models.settings import get_supabase_db +from packages.embeddings.vectors import Neurons from repository.files.upload_file import DocumentSerializable -from utils.vectors import Neurons @shared_task diff --git a/backend/models/brains.py b/backend/models/brains.py index c6b4500f4ea7..705d7afb221b 100644 --- a/backend/models/brains.py +++ b/backend/models/brains.py @@ -2,12 +2,11 @@ from uuid import UUID from logger import get_logger -from pydantic import BaseModel -from supabase.client import Client -from utils.vectors import get_unique_files_from_vector_ids - from models.databases.supabase.supabase import SupabaseDB from models.settings import get_supabase_client, get_supabase_db +from packages.embeddings.vectors import get_unique_files_from_vector_ids +from pydantic import BaseModel +from supabase.client import Client logger = get_logger(__name__) diff --git a/backend/utils/vectors.py b/backend/packages/embeddings/vectors.py similarity index 95% rename from backend/utils/vectors.py rename to backend/packages/embeddings/vectors.py index b1510c596671..039abca773d5 100644 --- a/backend/utils/vectors.py +++ b/backend/packages/embeddings/vectors.py @@ -10,6 +10,7 @@ logger = get_logger(__name__) +# TODO: Create interface for embeddings and implement it for Supabase and OpenAI (current Quivr) class Neurons(BaseModel): def create_vector(self, doc, user_openai_api_key=None): documents_vector_store = get_documents_vector_store() @@ -56,6 +57,7 @@ def process_batch(batch_ids: List[str]): logger.error("Error retrieving batched vectors", e) +# TODO: move to Knowledge class def get_unique_files_from_vector_ids(vectors_ids: List[str]): # Move into Vectors class """ diff --git a/backend/packages/files/parsers/github.py b/backend/packages/files/parsers/github.py index be1113611e3f..deb963eacc0c 100644 --- a/backend/packages/files/parsers/github.py +++ b/backend/packages/files/parsers/github.py @@ -5,8 +5,8 @@ from langchain.schema import Document from langchain.text_splitter import RecursiveCharacterTextSplitter from models import Brain, File +from packages.embeddings.vectors import Neurons from packages.files.file import compute_sha1_from_content -from utils.vectors import Neurons async def process_github( From 1a8db634cfe3487cea0ce7b7b7f9527b49296f50 Mon Sep 17 00:00:00 2001 From: gozineb Date: Tue, 14 Nov 2023 11:24:32 +0100 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=9A=9A=20move=20utils=20into=20packag?= =?UTF-8?q?es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/chat_service.py | 4 +++- backend/crawl_service.py | 2 +- backend/main.py | 4 ++-- backend/{ => packages}/utils/__init__.py | 0 .../utils/handle_request_validation_error.py | 0 backend/{ => packages}/utils/parse_message_time.py | 0 .../chat/get_chat_history_with_notifications.py | 9 ++------- backend/upload_service.py | 2 +- 8 files changed, 9 insertions(+), 12 deletions(-) rename backend/{ => packages}/utils/__init__.py (100%) rename backend/{ => packages}/utils/handle_request_validation_error.py (100%) rename backend/{ => packages}/utils/parse_message_time.py (100%) diff --git a/backend/chat_service.py b/backend/chat_service.py index 00a37b0d3c55..4f90d529b7da 100644 --- a/backend/chat_service.py +++ b/backend/chat_service.py @@ -1,6 +1,8 @@ import os -from utils import handle_request_validation_error +from packages.utils.handle_request_validation_error import ( + handle_request_validation_error, +) if __name__ == "__main__": # import needed here when running main.py to debug backend diff --git a/backend/crawl_service.py b/backend/crawl_service.py index 929811d84841..51faecbcd5f3 100644 --- a/backend/crawl_service.py +++ b/backend/crawl_service.py @@ -1,6 +1,6 @@ import os -from utils import handle_request_validation_error +from packages.utils import handle_request_validation_error if __name__ == "__main__": # import needed here when running main.py to debug backend diff --git a/backend/main.py b/backend/main.py index 970166ffc43b..4a6e6d6ab15e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,6 +1,6 @@ import os -from utils import handle_request_validation_error +from packages.utils import handle_request_validation_error if __name__ == "__main__": # import needed here when running main.py to debug backend @@ -17,6 +17,7 @@ from routes.api_key_routes import api_key_router from routes.brain_routes import brain_router from routes.chat_routes import chat_router +from routes.contact_routes import router as contact_router from routes.crawl_routes import crawl_router from routes.explore_routes import explore_router from routes.knowledge_routes import knowledge_router @@ -27,7 +28,6 @@ from routes.subscription_routes import subscription_router from routes.upload_routes import upload_router from routes.user_routes import user_router -from routes.contact_routes import router as contact_router logger = get_logger(__name__) diff --git a/backend/utils/__init__.py b/backend/packages/utils/__init__.py similarity index 100% rename from backend/utils/__init__.py rename to backend/packages/utils/__init__.py diff --git a/backend/utils/handle_request_validation_error.py b/backend/packages/utils/handle_request_validation_error.py similarity index 100% rename from backend/utils/handle_request_validation_error.py rename to backend/packages/utils/handle_request_validation_error.py diff --git a/backend/utils/parse_message_time.py b/backend/packages/utils/parse_message_time.py similarity index 100% rename from backend/utils/parse_message_time.py rename to backend/packages/utils/parse_message_time.py diff --git a/backend/repository/chat/get_chat_history_with_notifications.py b/backend/repository/chat/get_chat_history_with_notifications.py index de2ec18df8ec..87791f828ed1 100644 --- a/backend/repository/chat/get_chat_history_with_notifications.py +++ b/backend/repository/chat/get_chat_history_with_notifications.py @@ -3,15 +3,10 @@ from uuid import UUID from models.notifications import Notification +from packages.utils import parse_message_time from pydantic import BaseModel -from utils.parse_message_time import ( - parse_message_time, -) - from repository.chat.get_chat_history import GetChatHistoryOutput, get_chat_history -from repository.notification.get_chat_notifications import ( - get_chat_notifications, -) +from repository.notification.get_chat_notifications import get_chat_notifications class ChatItemType(Enum): diff --git a/backend/upload_service.py b/backend/upload_service.py index c746e687be31..4998a257a877 100644 --- a/backend/upload_service.py +++ b/backend/upload_service.py @@ -1,6 +1,6 @@ import os -from utils import handle_request_validation_error +from packages.utils import handle_request_validation_error if __name__ == "__main__": # import needed here when running main.py to debug backend From 990d362bed6a8858a64157f6ef383ffb62b50153 Mon Sep 17 00:00:00 2001 From: gozineb Date: Tue, 14 Nov 2023 11:31:57 +0100 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=9A=9A=20move=20loaders=20to=20files?= =?UTF-8?q?=20package?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/{ => packages/files}/loaders/__init__.py | 0 backend/{ => packages/files}/loaders/telegram.py | 0 backend/packages/files/parsers/telegram.py | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename backend/{ => packages/files}/loaders/__init__.py (100%) rename backend/{ => packages/files}/loaders/telegram.py (100%) diff --git a/backend/loaders/__init__.py b/backend/packages/files/loaders/__init__.py similarity index 100% rename from backend/loaders/__init__.py rename to backend/packages/files/loaders/__init__.py diff --git a/backend/loaders/telegram.py b/backend/packages/files/loaders/telegram.py similarity index 100% rename from backend/loaders/telegram.py rename to backend/packages/files/loaders/telegram.py diff --git a/backend/packages/files/parsers/telegram.py b/backend/packages/files/parsers/telegram.py index 51a4c86584c5..128ad7a85701 100644 --- a/backend/packages/files/parsers/telegram.py +++ b/backend/packages/files/parsers/telegram.py @@ -1,5 +1,5 @@ -from loaders.telegram import TelegramChatFileLoader from models import File +from packages.files.loaders.telegram import TelegramChatFileLoader from .common import process_file From 03906076bac965adf07a5880c0c8fbee44e4e294 Mon Sep 17 00:00:00 2001 From: gozineb Date: Tue, 14 Nov 2023 11:36:20 +0100 Subject: [PATCH 5/6] =?UTF-8?q?=F0=9F=9A=9A=20move=20auth=20folder=20in=20?= =?UTF-8?q?middlewares?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/{ => middlewares}/auth/__init__.py | 0 backend/{ => middlewares}/auth/api_key_handler.py | 0 backend/{ => middlewares}/auth/auth_bearer.py | 4 ++-- backend/{ => middlewares}/auth/jwt_token_handler.py | 0 backend/models/databases/supabase/vectors.py | 1 + backend/packages/embeddings/vectors.py | 8 -------- backend/routes/api_key_routes.py | 2 +- backend/routes/authorizations/brain_authorization.py | 3 +-- backend/routes/brain_routes.py | 3 +-- backend/routes/chat_routes.py | 3 +-- backend/routes/crawl_routes.py | 2 +- backend/routes/explore_routes.py | 2 +- backend/routes/knowledge_routes.py | 3 +-- backend/routes/notification_routes.py | 6 ++---- backend/routes/onboarding_routes.py | 8 ++++---- backend/routes/prompt_routes.py | 4 ++-- backend/routes/subscription_routes.py | 3 +-- backend/routes/upload_routes.py | 2 +- backend/routes/user_routes.py | 2 +- 19 files changed, 21 insertions(+), 35 deletions(-) rename backend/{ => middlewares}/auth/__init__.py (100%) rename backend/{ => middlewares}/auth/api_key_handler.py (100%) rename backend/{ => middlewares}/auth/auth_bearer.py (91%) rename backend/{ => middlewares}/auth/jwt_token_handler.py (100%) diff --git a/backend/auth/__init__.py b/backend/middlewares/auth/__init__.py similarity index 100% rename from backend/auth/__init__.py rename to backend/middlewares/auth/__init__.py diff --git a/backend/auth/api_key_handler.py b/backend/middlewares/auth/api_key_handler.py similarity index 100% rename from backend/auth/api_key_handler.py rename to backend/middlewares/auth/api_key_handler.py diff --git a/backend/auth/auth_bearer.py b/backend/middlewares/auth/auth_bearer.py similarity index 91% rename from backend/auth/auth_bearer.py rename to backend/middlewares/auth/auth_bearer.py index 93e4361bbfbb..9bb079d5a3fa 100644 --- a/backend/auth/auth_bearer.py +++ b/backend/middlewares/auth/auth_bearer.py @@ -1,10 +1,10 @@ import os from typing import Optional -from auth.api_key_handler import get_user_from_api_key, verify_api_key -from auth.jwt_token_handler import decode_access_token, verify_token from fastapi import Depends, HTTPException, Request from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from middlewares.auth.api_key_handler import get_user_from_api_key, verify_api_key +from middlewares.auth.jwt_token_handler import decode_access_token, verify_token from models import UserIdentity diff --git a/backend/auth/jwt_token_handler.py b/backend/middlewares/auth/jwt_token_handler.py similarity index 100% rename from backend/auth/jwt_token_handler.py rename to backend/middlewares/auth/jwt_token_handler.py diff --git a/backend/models/databases/supabase/vectors.py b/backend/models/databases/supabase/vectors.py index a17e7f004035..ed616368e559 100644 --- a/backend/models/databases/supabase/vectors.py +++ b/backend/models/databases/supabase/vectors.py @@ -40,6 +40,7 @@ def set_file_sha_from_metadata(self, file_sha1): return response + # TODO: remove duplicate similarity_search in supabase vector store def similarity_search(self, query_embedding, table, top_k, threshold): response = self.db.rpc( table, diff --git a/backend/packages/embeddings/vectors.py b/backend/packages/embeddings/vectors.py index 039abca773d5..19f9607a94d1 100644 --- a/backend/packages/embeddings/vectors.py +++ b/backend/packages/embeddings/vectors.py @@ -32,14 +32,6 @@ def create_embedding(self, content): embeddings = get_embeddings() return embeddings.embed_query(content) - def similarity_search(self, query, table="match_summaries", top_k=6, threshold=0.5): - query_embedding = self.create_embedding(query) - supabase_db = get_supabase_db() - summaries = supabase_db.similarity_search( - query_embedding, table, top_k, threshold - ) - return summaries.data - def error_callback(exception): print("An exception occurred:", exception) diff --git a/backend/routes/api_key_routes.py b/backend/routes/api_key_routes.py index e0fc34ef7a14..7e6ff85add7b 100644 --- a/backend/routes/api_key_routes.py +++ b/backend/routes/api_key_routes.py @@ -3,9 +3,9 @@ from uuid import uuid4 from asyncpg.exceptions import UniqueViolationError -from auth import AuthBearer, get_current_user from fastapi import APIRouter, Depends from logger import get_logger +from middlewares.auth import AuthBearer, get_current_user from models import UserIdentity, get_supabase_db from pydantic import BaseModel diff --git a/backend/routes/authorizations/brain_authorization.py b/backend/routes/authorizations/brain_authorization.py index 10b208afbf1b..403c5e4fd152 100644 --- a/backend/routes/authorizations/brain_authorization.py +++ b/backend/routes/authorizations/brain_authorization.py @@ -1,12 +1,11 @@ from typing import List, Optional, Union from uuid import UUID -from auth.auth_bearer import get_current_user from fastapi import Depends, HTTPException, status +from middlewares.auth.auth_bearer import get_current_user from models import UserIdentity from repository.brain import get_brain_for_user from repository.brain.get_brain_details import get_brain_details - from routes.authorizations.types import RoleEnum diff --git a/backend/routes/brain_routes.py b/backend/routes/brain_routes.py index 4f331664aa4b..d9537040d948 100644 --- a/backend/routes/brain_routes.py +++ b/backend/routes/brain_routes.py @@ -1,8 +1,8 @@ from uuid import UUID -from auth import AuthBearer, get_current_user from fastapi import APIRouter, Depends, HTTPException from logger import get_logger +from middlewares.auth.auth_bearer import AuthBearer, get_current_user from models import UserIdentity, UserUsage from models.brain_entity import PublicBrain from models.databases.supabase.brains import ( @@ -24,7 +24,6 @@ update_brain_by_id, ) from repository.prompt import delete_prompt_by_id, get_prompt_by_id - from routes.authorizations.brain_authorization import has_brain_authorization from routes.authorizations.types import RoleEnum diff --git a/backend/routes/chat_routes.py b/backend/routes/chat_routes.py index 088c0967128d..d14c9044ad75 100644 --- a/backend/routes/chat_routes.py +++ b/backend/routes/chat_routes.py @@ -2,11 +2,11 @@ from uuid import UUID from venv import logger -from auth import AuthBearer, get_current_user from fastapi import APIRouter, Depends, HTTPException, Query, Request from fastapi.responses import StreamingResponse from llm.qa_base import QABaseBrainPicking from llm.qa_headless import HeadlessQA +from middlewares.auth import AuthBearer, get_current_user from models import ( Brain, BrainEntity, @@ -32,7 +32,6 @@ get_chat_history_with_notifications, ) from repository.notification.remove_chat_notifications import remove_chat_notifications - from routes.chat.factory import get_chat_strategy from routes.chat.utils import ( NullableUUID, diff --git a/backend/routes/crawl_routes.py b/backend/routes/crawl_routes.py index 1f5b1f6e039f..27520181eaee 100644 --- a/backend/routes/crawl_routes.py +++ b/backend/routes/crawl_routes.py @@ -1,11 +1,11 @@ from typing import Optional from uuid import UUID -from auth import AuthBearer, get_current_user from celery_worker import process_crawl_and_notify from crawl.crawler import CrawlWebsite from fastapi import APIRouter, Depends, Query, Request from logger import get_logger +from middlewares.auth import AuthBearer, get_current_user from models import Brain, UserIdentity, UserUsage from models.databases.supabase.knowledge import CreateKnowledgeProperties from models.databases.supabase.notifications import CreateNotificationProperties diff --git a/backend/routes/explore_routes.py b/backend/routes/explore_routes.py index 7497f0032663..67f6cbf2f0b9 100644 --- a/backend/routes/explore_routes.py +++ b/backend/routes/explore_routes.py @@ -1,7 +1,7 @@ from uuid import UUID -from auth import AuthBearer, get_current_user from fastapi import APIRouter, Depends, Query +from middlewares.auth import AuthBearer, get_current_user from models import Brain, UserIdentity, get_supabase_db from routes.authorizations.brain_authorization import ( RoleEnum, diff --git a/backend/routes/knowledge_routes.py b/backend/routes/knowledge_routes.py index 27a42ca778ba..7699a802efec 100644 --- a/backend/routes/knowledge_routes.py +++ b/backend/routes/knowledge_routes.py @@ -1,15 +1,14 @@ from uuid import UUID -from auth import AuthBearer, get_current_user from fastapi import APIRouter, Depends, Query from logger import get_logger +from middlewares.auth import AuthBearer, get_current_user from models import Brain, UserIdentity from repository.files.delete_file import delete_file_from_storage from repository.files.generate_file_signed_url import generate_file_signed_url from repository.knowledge.get_all_knowledge import get_all_knowledge from repository.knowledge.get_knowledge import get_knowledge from repository.knowledge.remove_knowledge import remove_knowledge - from routes.authorizations.brain_authorization import ( RoleEnum, has_brain_authorization, diff --git a/backend/routes/notification_routes.py b/backend/routes/notification_routes.py index 250938e51816..6483a641b694 100644 --- a/backend/routes/notification_routes.py +++ b/backend/routes/notification_routes.py @@ -1,10 +1,8 @@ from uuid import UUID -from auth import AuthBearer from fastapi import APIRouter, Depends -from repository.notification.get_chat_notifications import ( - get_chat_notifications, -) +from middlewares.auth import AuthBearer +from repository.notification.get_chat_notifications import get_chat_notifications notification_router = APIRouter() diff --git a/backend/routes/onboarding_routes.py b/backend/routes/onboarding_routes.py index 9e928d8c29b8..ed1ff2622fd6 100644 --- a/backend/routes/onboarding_routes.py +++ b/backend/routes/onboarding_routes.py @@ -1,8 +1,8 @@ -from auth import ( - AuthBearer, - get_current_user, # Assuming you have a get_current_user function -) from fastapi import APIRouter, Depends +from middlewares.auth import ( + get_current_user, +) # Assuming you have a get_current_user function +from middlewares.auth import AuthBearer from models.databases.supabase.onboarding import ( OnboardingStates, OnboardingUpdatableProperties, diff --git a/backend/routes/prompt_routes.py b/backend/routes/prompt_routes.py index 76482195df46..45dce9d20ab4 100644 --- a/backend/routes/prompt_routes.py +++ b/backend/routes/prompt_routes.py @@ -1,12 +1,12 @@ from uuid import UUID -from auth import AuthBearer from fastapi import APIRouter, Depends +from middlewares.auth import AuthBearer +from models import Prompt from models.databases.supabase.prompts import ( CreatePromptProperties, PromptUpdatableProperties, ) -from models import Prompt from repository.prompt import ( create_prompt, get_prompt_by_id, diff --git a/backend/routes/subscription_routes.py b/backend/routes/subscription_routes.py index f6531296a60a..c42ac37a20b4 100644 --- a/backend/routes/subscription_routes.py +++ b/backend/routes/subscription_routes.py @@ -1,8 +1,8 @@ from typing import List from uuid import UUID -from auth.auth_bearer import AuthBearer, get_current_user from fastapi import APIRouter, Depends, HTTPException +from middlewares.auth.auth_bearer import AuthBearer, get_current_user from models import BrainSubscription, PromptStatusEnum, UserIdentity from pydantic import BaseModel from repository.brain import ( @@ -21,7 +21,6 @@ ) from repository.prompt import delete_prompt_by_id, get_prompt_by_id from repository.user import get_user_id_by_user_email - from routes.authorizations.brain_authorization import ( RoleEnum, has_brain_authorization, diff --git a/backend/routes/upload_routes.py b/backend/routes/upload_routes.py index d31ff5c1f8bc..c65f6fc40038 100644 --- a/backend/routes/upload_routes.py +++ b/backend/routes/upload_routes.py @@ -2,10 +2,10 @@ from typing import Optional from uuid import UUID -from auth import AuthBearer, get_current_user from celery_worker import process_file_and_notify from fastapi import APIRouter, Depends, HTTPException, Query, Request, UploadFile from logger import get_logger +from middlewares.auth import AuthBearer, get_current_user from models import Brain, UserIdentity, UserUsage from models.databases.supabase.knowledge import CreateKnowledgeProperties from models.databases.supabase.notifications import CreateNotificationProperties diff --git a/backend/routes/user_routes.py b/backend/routes/user_routes.py index 1beecdcfc4f6..2b5e683e9943 100644 --- a/backend/routes/user_routes.py +++ b/backend/routes/user_routes.py @@ -1,7 +1,7 @@ import time -from auth import AuthBearer, get_current_user from fastapi import APIRouter, Depends, Request +from middlewares.auth import AuthBearer, get_current_user from models import Brain, UserIdentity, UserUsage from repository.brain import get_user_default_brain from repository.user_identity.get_user_identity import get_user_identity From 5beec4eeee5125ed5a77c93514fa4333dc997326 Mon Sep 17 00:00:00 2001 From: gozineb Date: Tue, 14 Nov 2023 11:36:38 +0100 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=9A=9A=20move=20crawl=20to=20files=20?= =?UTF-8?q?package?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/celery_worker.py | 2 +- backend/{ => packages/files}/crawl/__init__.py | 0 backend/{ => packages/files}/crawl/crawler.py | 0 backend/routes/crawl_routes.py | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) rename backend/{ => packages/files}/crawl/__init__.py (100%) rename backend/{ => packages/files}/crawl/crawler.py (100%) diff --git a/backend/celery_worker.py b/backend/celery_worker.py index e5d6a365fc2a..663648fb62be 100644 --- a/backend/celery_worker.py +++ b/backend/celery_worker.py @@ -4,12 +4,12 @@ from celery import Celery from celery.schedules import crontab -from crawl.crawler import CrawlWebsite from fastapi import UploadFile from models.databases.supabase.notifications import NotificationUpdatableProperties from models.files import File from models.notifications import NotificationsStatusEnum from models.settings import get_supabase_client +from packages.files.crawl.crawler import CrawlWebsite from packages.files.parsers.github import process_github from packages.files.processors import filter_file from repository.brain.update_brain_last_update_time import update_brain_last_update_time diff --git a/backend/crawl/__init__.py b/backend/packages/files/crawl/__init__.py similarity index 100% rename from backend/crawl/__init__.py rename to backend/packages/files/crawl/__init__.py diff --git a/backend/crawl/crawler.py b/backend/packages/files/crawl/crawler.py similarity index 100% rename from backend/crawl/crawler.py rename to backend/packages/files/crawl/crawler.py diff --git a/backend/routes/crawl_routes.py b/backend/routes/crawl_routes.py index 27520181eaee..9328d7ee2ff3 100644 --- a/backend/routes/crawl_routes.py +++ b/backend/routes/crawl_routes.py @@ -2,7 +2,6 @@ from uuid import UUID from celery_worker import process_crawl_and_notify -from crawl.crawler import CrawlWebsite from fastapi import APIRouter, Depends, Query, Request from logger import get_logger from middlewares.auth import AuthBearer, get_current_user @@ -10,6 +9,7 @@ from models.databases.supabase.knowledge import CreateKnowledgeProperties from models.databases.supabase.notifications import CreateNotificationProperties from models.notifications import NotificationsStatusEnum +from packages.files.crawl.crawler import CrawlWebsite from packages.files.file import convert_bytes from repository.knowledge.add_knowledge import add_knowledge from repository.notification.add_notification import add_notification