From 9766befb5372ac734a383ff227f04b12edc9e9cc Mon Sep 17 00:00:00 2001 From: Zineb El Bachiri <100568984+gozineb@users.noreply.github.com> Date: Wed, 29 Nov 2023 09:04:03 +0100 Subject: [PATCH] refactor: knowledge module (#1743) # Description - Refactor knowledge to a module - This PR breaks the Github Processor function -> need to comment brain and file imports as it creates a circular dependency issue. Should be fixed and reverted in next PR. --- .vscode/settings.json | 1 + backend/main.py | 2 +- backend/models/databases/repository.py | 20 ------- backend/models/databases/supabase/__init__.py | 1 - backend/models/databases/supabase/supabase.py | 3 - .../knowledge/__init__.py | 0 .../modules/knowledge/controller/__init__.py | 1 + .../knowledge/controller}/knowledge_routes.py | 15 +++-- backend/modules/knowledge/dto/__init__.py | 2 + backend/modules/knowledge/dto/inputs.py | 16 +++++ backend/modules/knowledge/dto/outputs.py | 8 +++ backend/modules/knowledge/entity/__init__.py | 1 + .../knowledge/entity}/knowledge.py | 0 .../modules/knowledge/repository/__init__.py | 1 + .../repository/knowledge_interface.py | 58 +++++++++++++++++++ .../knowledge/repository/knowledges.py} | 44 +++++--------- backend/modules/knowledge/service/__init__.py | 0 .../knowledge/service/knowledge_service.py | 46 +++++++++++++++ backend/packages/files/parsers/github.py | 33 ++++++----- backend/repository/brain/delete_brain.py | 9 ++- backend/repository/knowledge/add_knowledge.py | 14 ----- .../repository/knowledge/get_all_knowledge.py | 14 ----- backend/repository/knowledge/get_knowledge.py | 15 ----- .../knowledge/remove_brain_all_knowledge.py | 14 ----- .../repository/knowledge/remove_knowledge.py | 16 ----- backend/routes/crawl_routes.py | 7 ++- backend/routes/upload_routes.py | 7 ++- 27 files changed, 186 insertions(+), 162 deletions(-) rename backend/{repository => modules}/knowledge/__init__.py (100%) create mode 100644 backend/modules/knowledge/controller/__init__.py rename backend/{routes => modules/knowledge/controller}/knowledge_routes.py (88%) create mode 100644 backend/modules/knowledge/dto/__init__.py create mode 100644 backend/modules/knowledge/dto/inputs.py create mode 100644 backend/modules/knowledge/dto/outputs.py create mode 100644 backend/modules/knowledge/entity/__init__.py rename backend/{models => modules/knowledge/entity}/knowledge.py (100%) create mode 100644 backend/modules/knowledge/repository/__init__.py create mode 100644 backend/modules/knowledge/repository/knowledge_interface.py rename backend/{models/databases/supabase/knowledge.py => modules/knowledge/repository/knowledges.py} (68%) create mode 100644 backend/modules/knowledge/service/__init__.py create mode 100644 backend/modules/knowledge/service/knowledge_service.py delete mode 100644 backend/repository/knowledge/add_knowledge.py delete mode 100644 backend/repository/knowledge/get_all_knowledge.py delete mode 100644 backend/repository/knowledge/get_knowledge.py delete mode 100644 backend/repository/knowledge/remove_brain_all_knowledge.py delete mode 100644 backend/repository/knowledge/remove_knowledge.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 93e0a4a0ab41..bce7e19de0a0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,6 +7,7 @@ }, "python.linting.enabled": true, "python.linting.flake8Enabled": true, + "python.analysis.extraPaths": ["./backend"], "editor.formatOnSave": true, "[python]": { "editor.defaultFormatter": "ms-python.black-formatter", diff --git a/backend/main.py b/backend/main.py index 89041666327d..e989c7e73dd2 100644 --- a/backend/main.py +++ b/backend/main.py @@ -13,6 +13,7 @@ from fastapi.responses import JSONResponse from logger import get_logger from middlewares.cors import add_cors_middleware +from modules.knowledge.controller.knowledge_routes import knowledge_router from modules.notification.controller.notification_routes import notification_router from modules.onboarding.controller.onboarding_routes import onboarding_router from modules.prompt.controller.prompt_routes import prompt_router @@ -22,7 +23,6 @@ from routes.chat_routes import chat_router from routes.contact_routes import router as contact_router from routes.crawl_routes import crawl_router -from routes.knowledge_routes import knowledge_router from routes.misc_routes import misc_router from routes.subscription_routes import subscription_router from routes.upload_routes import upload_router diff --git a/backend/models/databases/repository.py b/backend/models/databases/repository.py index 136e91b30a11..839ba86ad14c 100644 --- a/backend/models/databases/repository.py +++ b/backend/models/databases/repository.py @@ -204,26 +204,6 @@ def get_vectors_in_batch(self, batch_ids): def get_vectors_by_file_sha1(self, file_sha1): pass - @abstractmethod - def insert_knowledge(self, brain_id: UUID): - pass - - @abstractmethod - def remove_knowledge_by_id(self, knowledge_id: UUID): - pass - - @abstractmethod - def remove_brain_all_knowledge(self, brain_id: UUID): - pass - - @abstractmethod - def get_knowledge_by_id(self, knowledge_id: UUID): - pass - - @abstractmethod - def get_all_knowledge_in_brain(self, brain_id: UUID): - pass - @abstractmethod def get_api_brain_definition(self, brain_id: UUID): pass diff --git a/backend/models/databases/supabase/__init__.py b/backend/models/databases/supabase/__init__.py index 7e3022605a14..863e22eb9057 100644 --- a/backend/models/databases/supabase/__init__.py +++ b/backend/models/databases/supabase/__init__.py @@ -5,6 +5,5 @@ BrainSubscription from models.databases.supabase.chats import Chats from models.databases.supabase.files import File -from models.databases.supabase.knowledge import Knowledges from models.databases.supabase.user_usage import UserUsage from models.databases.supabase.vectors import Vector diff --git a/backend/models/databases/supabase/supabase.py b/backend/models/databases/supabase/supabase.py index 9693a97acb1b..6d4c4a607041 100644 --- a/backend/models/databases/supabase/supabase.py +++ b/backend/models/databases/supabase/supabase.py @@ -6,7 +6,6 @@ BrainSubscription, Chats, File, - Knowledges, UserUsage, Vector, ) @@ -22,7 +21,6 @@ class SupabaseDB( ApiKeyHandler, Chats, Vector, - Knowledges, ApiBrainDefinitions, ): def __init__(self, supabase_client): @@ -34,5 +32,4 @@ def __init__(self, supabase_client): ApiKeyHandler.__init__(self, supabase_client) Chats.__init__(self, supabase_client) Vector.__init__(self, supabase_client) - Knowledges.__init__(self, supabase_client) ApiBrainDefinitions.__init__(self, supabase_client) diff --git a/backend/repository/knowledge/__init__.py b/backend/modules/knowledge/__init__.py similarity index 100% rename from backend/repository/knowledge/__init__.py rename to backend/modules/knowledge/__init__.py diff --git a/backend/modules/knowledge/controller/__init__.py b/backend/modules/knowledge/controller/__init__.py new file mode 100644 index 000000000000..23c692c11761 --- /dev/null +++ b/backend/modules/knowledge/controller/__init__.py @@ -0,0 +1 @@ +from .knowledge_routes import knowledge_router \ No newline at end of file diff --git a/backend/routes/knowledge_routes.py b/backend/modules/knowledge/controller/knowledge_routes.py similarity index 88% rename from backend/routes/knowledge_routes.py rename to backend/modules/knowledge/controller/knowledge_routes.py index fa4ca43fe754..91e6a73779da 100644 --- a/backend/routes/knowledge_routes.py +++ b/backend/modules/knowledge/controller/knowledge_routes.py @@ -4,13 +4,10 @@ from logger import get_logger from middlewares.auth import AuthBearer, get_current_user from models import Brain +from modules.knowledge.service.knowledge_service import KnowledgeService from modules.user.entity.user_identity import UserIdentity from repository.files.delete_file import delete_file_from_storage from repository.files.generate_file_signed_url import generate_file_signed_url -from repository.knowledge.get_all_knowledge import get_all_knowledge -from repository.knowledge.get_knowledge import get_knowledge -from repository.knowledge.remove_knowledge import remove_knowledge - from routes.authorizations.brain_authorization import ( RoleEnum, has_brain_authorization, @@ -20,6 +17,8 @@ knowledge_router = APIRouter() logger = get_logger(__name__) +knowledge_service = KnowledgeService() + @knowledge_router.get( "/knowledge", dependencies=[Depends(AuthBearer())], tags=["Knowledge"] @@ -34,7 +33,7 @@ async def list_knowledge_in_brain_endpoint( validate_brain_authorization(brain_id=brain_id, user_id=current_user.id) - knowledges = get_all_knowledge(brain_id) + knowledges = knowledge_service.get_all_knowledge(brain_id) logger.info(f"List of knowledge from knowledge table: {knowledges}") return {"knowledges": knowledges} @@ -59,9 +58,9 @@ async def delete_endpoint( brain = Brain(id=brain_id) - knowledge = get_knowledge(knowledge_id) + knowledge = knowledge_service.get_knowledge(knowledge_id) file_name = knowledge.file_name if knowledge.file_name else knowledge.url - remove_knowledge(knowledge_id) + knowledge_service.remove_knowledge(knowledge_id) if knowledge.file_name: delete_file_from_storage(f"{brain_id}/{knowledge.file_name}") @@ -87,7 +86,7 @@ async def generate_signed_url_endpoint( Generate a signed url to download the file from storage. """ - knowledge = get_knowledge(knowledge_id) + knowledge = knowledge_service.get_knowledge(knowledge_id) validate_brain_authorization(brain_id=knowledge.brain_id, user_id=current_user.id) diff --git a/backend/modules/knowledge/dto/__init__.py b/backend/modules/knowledge/dto/__init__.py new file mode 100644 index 000000000000..4f3a4b9f7bb5 --- /dev/null +++ b/backend/modules/knowledge/dto/__init__.py @@ -0,0 +1,2 @@ +from .inputs import CreateKnowledgeProperties +from .outputs import DeleteKnowledgeResponse diff --git a/backend/modules/knowledge/dto/inputs.py b/backend/modules/knowledge/dto/inputs.py new file mode 100644 index 000000000000..18184d3dec7b --- /dev/null +++ b/backend/modules/knowledge/dto/inputs.py @@ -0,0 +1,16 @@ +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel + + +class CreateKnowledgeProperties(BaseModel): + brain_id: UUID + file_name: Optional[str] = None + url: Optional[str] = None + extension: str = "txt" + + def dict(self, *args, **kwargs): + knowledge_dict = super().dict(*args, **kwargs) + knowledge_dict["brain_id"] = str(knowledge_dict.get("brain_id")) + return knowledge_dict diff --git a/backend/modules/knowledge/dto/outputs.py b/backend/modules/knowledge/dto/outputs.py new file mode 100644 index 000000000000..905da7cf0d7f --- /dev/null +++ b/backend/modules/knowledge/dto/outputs.py @@ -0,0 +1,8 @@ +from uuid import UUID + +from pydantic import BaseModel + + +class DeleteKnowledgeResponse(BaseModel): + status: str = "delete" + knowledge_id: UUID diff --git a/backend/modules/knowledge/entity/__init__.py b/backend/modules/knowledge/entity/__init__.py new file mode 100644 index 000000000000..6f7620c446d2 --- /dev/null +++ b/backend/modules/knowledge/entity/__init__.py @@ -0,0 +1 @@ +from .knowledge import Knowledge diff --git a/backend/models/knowledge.py b/backend/modules/knowledge/entity/knowledge.py similarity index 100% rename from backend/models/knowledge.py rename to backend/modules/knowledge/entity/knowledge.py diff --git a/backend/modules/knowledge/repository/__init__.py b/backend/modules/knowledge/repository/__init__.py new file mode 100644 index 000000000000..a69273cf5473 --- /dev/null +++ b/backend/modules/knowledge/repository/__init__.py @@ -0,0 +1 @@ +from .knowledges import Knowledges diff --git a/backend/modules/knowledge/repository/knowledge_interface.py b/backend/modules/knowledge/repository/knowledge_interface.py new file mode 100644 index 000000000000..beaac48d41e8 --- /dev/null +++ b/backend/modules/knowledge/repository/knowledge_interface.py @@ -0,0 +1,58 @@ +from abc import ABC, abstractmethod +from typing import List +from uuid import UUID + +from modules.knowledge.dto.inputs import CreateKnowledgeProperties +from modules.knowledge.dto.outputs import DeleteKnowledgeResponse +from modules.knowledge.entity.knowledge import Knowledge + + +class KnowledgeInterface(ABC): + @abstractmethod + def insert_knowledge(self, knowledge: CreateKnowledgeProperties) -> Knowledge: + """ + Add a knowledge + """ + pass + + @abstractmethod + def remove_knowledge_by_id( + # todo: update remove brain endpoints to first delete the knowledge + self, + knowledge_id: UUID, + ) -> DeleteKnowledgeResponse: + """ + Args: + knowledge_id (UUID): The id of the knowledge + + Returns: + str: Status message + """ + pass + + @abstractmethod + def get_knowledge_by_id(self, knowledge_id: UUID) -> Knowledge: + """ + Get a knowledge by its id + Args: + brain_id (UUID): The id of the brain + """ + pass + + @abstractmethod + def get_all_knowledge_in_brain(self, brain_id: UUID) -> List[Knowledge]: + """ + Get all the knowledge in a brain + Args: + brain_id (UUID): The id of the brain + """ + pass + + @abstractmethod + def remove_brain_all_knowledge(self, brain_id: UUID) -> None: + """ + Remove all knowledge in a brain + Args: + brain_id (UUID): The id of the brain + """ + pass diff --git a/backend/models/databases/supabase/knowledge.py b/backend/modules/knowledge/repository/knowledges.py similarity index 68% rename from backend/models/databases/supabase/knowledge.py rename to backend/modules/knowledge/repository/knowledges.py index 8c54c5264118..8623bb02aa17 100644 --- a/backend/models/databases/supabase/knowledge.py +++ b/backend/modules/knowledge/repository/knowledges.py @@ -1,34 +1,16 @@ -from typing import List, Optional -from uuid import UUID - from fastapi import HTTPException -from models.databases.repository import Repository -from models.knowledge import Knowledge -from pydantic import BaseModel - - -class CreateKnowledgeProperties(BaseModel): - brain_id: UUID - file_name: Optional[str] = None - url: Optional[str] = None - extension: str = "txt" - - def dict(self, *args, **kwargs): - knowledge_dict = super().dict(*args, **kwargs) - knowledge_dict["brain_id"] = str(knowledge_dict.get("brain_id")) - return knowledge_dict - - -class DeleteKnowledgeResponse(BaseModel): - status: str = "delete" - knowledge_id: UUID +from models.settings import get_supabase_client +from modules.knowledge.dto.outputs import DeleteKnowledgeResponse +from modules.knowledge.entity.knowledge import Knowledge +from modules.knowledge.repository.knowledge_interface import KnowledgeInterface -class Knowledges(Repository): - def __init__(self, supabase_client): +class Knowledges(KnowledgeInterface): + def __init__(self): + supabase_client = get_supabase_client() self.db = supabase_client - def insert_knowledge(self, knowledge: CreateKnowledgeProperties) -> Knowledge: + def insert_knowledge(self, knowledge): """ Add a knowledge """ @@ -38,8 +20,8 @@ def insert_knowledge(self, knowledge: CreateKnowledgeProperties) -> Knowledge: def remove_knowledge_by_id( # todo: update remove brain endpoints to first delete the knowledge self, - knowledge_id: UUID, - ) -> DeleteKnowledgeResponse: + knowledge_id, + ): """ Args: knowledge_id (UUID): The id of the knowledge @@ -64,7 +46,7 @@ def remove_knowledge_by_id( knowledge_id=knowledge_id, ) - def get_knowledge_by_id(self, knowledge_id: UUID) -> Knowledge: + def get_knowledge_by_id(self, knowledge_id): """ Get a knowledge by its id Args: @@ -79,7 +61,7 @@ def get_knowledge_by_id(self, knowledge_id: UUID) -> Knowledge: return Knowledge(**knowledge[0]) - def get_all_knowledge_in_brain(self, brain_id: UUID) -> List[Knowledge]: + def get_all_knowledge_in_brain(self, brain_id): """ Get all the knowledge in a brain Args: @@ -94,7 +76,7 @@ def get_all_knowledge_in_brain(self, brain_id: UUID) -> List[Knowledge]: return [Knowledge(**knowledge) for knowledge in all_knowledge] - def remove_brain_all_knowledge(self, brain_id: UUID) -> None: + def remove_brain_all_knowledge(self, brain_id): """ Remove all knowledge in a brain Args: diff --git a/backend/modules/knowledge/service/__init__.py b/backend/modules/knowledge/service/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/modules/knowledge/service/knowledge_service.py b/backend/modules/knowledge/service/knowledge_service.py new file mode 100644 index 000000000000..00125fcd56d9 --- /dev/null +++ b/backend/modules/knowledge/service/knowledge_service.py @@ -0,0 +1,46 @@ +from uuid import UUID + +from logger import get_logger +from modules.knowledge.dto.inputs import CreateKnowledgeProperties +from modules.knowledge.entity.knowledge import Knowledge +from modules.knowledge.repository.knowledge_interface import KnowledgeInterface +from modules.knowledge.repository.knowledges import Knowledges + +logger = get_logger(__name__) + + +class KnowledgeService: + repository: KnowledgeInterface + + def __init__(self): + self.repository = Knowledges() + + def add_knowledge(self, knowledge_to_add: CreateKnowledgeProperties): + knowledge = self.repository.insert_knowledge(knowledge_to_add) + + logger.info(f"Knowledge { knowledge.id} added successfully") + return knowledge + + def get_all_knowledge(self, brain_id: UUID): + knowledges = self.repository.get_all_knowledge_in_brain(brain_id) + + return knowledges + + def get_knowledge(self, knowledge_id: UUID) -> Knowledge: + knowledge = self.repository.get_knowledge_by_id(knowledge_id) + + return knowledge + + def remove_brain_all_knowledge(self, brain_id: UUID) -> None: + self.repository.remove_brain_all_knowledge(brain_id) + + logger.info( + f"All knowledge in brain {brain_id} removed successfully from table" + ) + + def remove_knowledge(self, knowledge_id: UUID): + message = self.repository.remove_knowledge_by_id(knowledge_id) + + logger.info(f"Knowledge { knowledge_id} removed successfully from table") + + return message diff --git a/backend/packages/files/parsers/github.py b/backend/packages/files/parsers/github.py index bd15296c24f8..9eb75d065655 100644 --- a/backend/packages/files/parsers/github.py +++ b/backend/packages/files/parsers/github.py @@ -4,8 +4,9 @@ from langchain.document_loaders import GitLoader from langchain.schema import Document from langchain.text_splitter import RecursiveCharacterTextSplitter -from models import Brain, File -from packages.embeddings.vectors import Neurons + +# from models import Brain, File +# from packages.embeddings.vectors import Neurons from packages.files.file import compute_sha1_from_content @@ -55,22 +56,26 @@ async def process_github( } doc_with_metadata = Document(page_content=doc.page_content, metadata=metadata) - file = File( - file_sha1=compute_sha1_from_content(doc.page_content.encode("utf-8")) - ) + print(doc_with_metadata.metadata["file_name"]) + + # TO FIX: Import of file and brain creates a circular dependency + # file = File( + # file_sha1=compute_sha1_from_content(doc.page_content.encode("utf-8")) + # ) - file_exists = file.file_already_exists() + # file_exists = file.file_already_exists() - if not file_exists: - neurons = Neurons() - created_vector = neurons.create_vector(doc_with_metadata) + # if not file_exists: + # neurons = Neurons() + # created_vector = neurons.create_vector(doc_with_metadata) - file_exists_in_brain = file.file_already_exists_in_brain(brain_id) + # file_exists_in_brain = file.file_already_exists_in_brain(brain_id) - if not file_exists_in_brain: - brain = Brain(id=brain_id) - file.link_file_to_brain(brain) + # if not file_exists_in_brain: + # brain = Brain(id=brain_id) + # file.link_file_to_brain(brain) return { - "message": f"✅ Github with {len(documents)} files has been uploaded.", + # "message": f"✅ Github with {len(documents)} files has been uploaded.", + "message": "Github processor is currently unavailable.", "type": "success", } diff --git a/backend/repository/brain/delete_brain.py b/backend/repository/brain/delete_brain.py index 53f9b8d04793..9b3844629055 100644 --- a/backend/repository/brain/delete_brain.py +++ b/backend/repository/brain/delete_brain.py @@ -3,15 +3,14 @@ from fastapi import HTTPException from models.brain_entity import BrainType from models.settings import get_supabase_db - +from modules.knowledge.service.knowledge_service import KnowledgeService from repository.api_brain_definition.delete_api_brain_definition import ( delete_api_brain_definition, ) from repository.brain import get_brain_by_id from repository.brain.delete_brain_secrets import delete_brain_secrets_values -from repository.knowledge.remove_brain_all_knowledge import ( - remove_brain_all_knowledge, -) + +knowledge_service = KnowledgeService() def delete_brain(brain_id: UUID) -> dict[str, str]: @@ -27,7 +26,7 @@ def delete_brain(brain_id: UUID) -> dict[str, str]: ) delete_api_brain_definition(brain_id=brain_id) else: - remove_brain_all_knowledge(brain_id) + knowledge_service.remove_brain_all_knowledge(brain_id) supabase_db.delete_brain_vector(str(brain_id)) supabase_db.delete_brain_users(str(brain_id)) diff --git a/backend/repository/knowledge/add_knowledge.py b/backend/repository/knowledge/add_knowledge.py deleted file mode 100644 index bcae311f250a..000000000000 --- a/backend/repository/knowledge/add_knowledge.py +++ /dev/null @@ -1,14 +0,0 @@ -from logger import get_logger -from models.databases.supabase.knowledge import CreateKnowledgeProperties -from models.settings import get_supabase_db - -logger = get_logger(__name__) - - -def add_knowledge(knowledge_to_add: CreateKnowledgeProperties): - supabase_db = get_supabase_db() - - knowledge = supabase_db.insert_knowledge(knowledge_to_add) - - logger.info(f"Knowledge { knowledge.id} added successfully") - return knowledge diff --git a/backend/repository/knowledge/get_all_knowledge.py b/backend/repository/knowledge/get_all_knowledge.py deleted file mode 100644 index 39f636d3b35a..000000000000 --- a/backend/repository/knowledge/get_all_knowledge.py +++ /dev/null @@ -1,14 +0,0 @@ -from uuid import UUID - -from logger import get_logger -from models.settings import get_supabase_db - -logger = get_logger(__name__) - - -def get_all_knowledge(brain_id: UUID): - supabase_db = get_supabase_db() - - knowledges = supabase_db.get_all_knowledge_in_brain(brain_id) - - return knowledges diff --git a/backend/repository/knowledge/get_knowledge.py b/backend/repository/knowledge/get_knowledge.py deleted file mode 100644 index 273fb5646e2f..000000000000 --- a/backend/repository/knowledge/get_knowledge.py +++ /dev/null @@ -1,15 +0,0 @@ -from uuid import UUID - -from logger import get_logger -from models.knowledge import Knowledge -from models.settings import get_supabase_db - -logger = get_logger(__name__) - - -def get_knowledge(knowledge_id: UUID) -> Knowledge: - supabase_db = get_supabase_db() - - knowledge = supabase_db.get_knowledge_by_id(knowledge_id) - - return knowledge diff --git a/backend/repository/knowledge/remove_brain_all_knowledge.py b/backend/repository/knowledge/remove_brain_all_knowledge.py deleted file mode 100644 index f01302ccb3b6..000000000000 --- a/backend/repository/knowledge/remove_brain_all_knowledge.py +++ /dev/null @@ -1,14 +0,0 @@ -from uuid import UUID - -from logger import get_logger -from models.settings import get_supabase_db - -logger = get_logger(__name__) - - -def remove_brain_all_knowledge(brain_id: UUID) -> None: - supabase_db = get_supabase_db() - - supabase_db.remove_brain_all_knowledge(brain_id) - - logger.info(f"All knowledge in brain {brain_id} removed successfully from table") diff --git a/backend/repository/knowledge/remove_knowledge.py b/backend/repository/knowledge/remove_knowledge.py deleted file mode 100644 index 0ba307ed0a24..000000000000 --- a/backend/repository/knowledge/remove_knowledge.py +++ /dev/null @@ -1,16 +0,0 @@ -from uuid import UUID - -from logger import get_logger -from models.settings import get_supabase_db - -logger = get_logger(__name__) - - -def remove_knowledge(knowledge_id: UUID): - supabase_db = get_supabase_db() - - message = supabase_db.remove_knowledge_by_id(knowledge_id) - - logger.info(f"Knowledge { knowledge_id} removed successfully from table") - - return message diff --git a/backend/routes/crawl_routes.py b/backend/routes/crawl_routes.py index c633a2b32263..ce96185738d4 100644 --- a/backend/routes/crawl_routes.py +++ b/backend/routes/crawl_routes.py @@ -6,19 +6,20 @@ from logger import get_logger from middlewares.auth import AuthBearer, get_current_user from models import Brain, UserUsage -from models.databases.supabase.knowledge import CreateKnowledgeProperties +from modules.knowledge.dto.inputs import CreateKnowledgeProperties +from modules.knowledge.service.knowledge_service import KnowledgeService from modules.notification.dto.inputs import CreateNotificationProperties from modules.notification.entity.notification import NotificationsStatusEnum from modules.notification.service.notification_service import NotificationService from modules.user.entity.user_identity import UserIdentity from packages.files.crawl.crawler import CrawlWebsite from packages.files.file import convert_bytes -from repository.knowledge.add_knowledge import add_knowledge logger = get_logger(__name__) crawl_router = APIRouter() notification_service = NotificationService() +knowledge_service = KnowledgeService() @crawl_router.get("/crawl/healthz", tags=["Health"]) @@ -72,7 +73,7 @@ async def crawl_endpoint( extension="html", ) - added_knowledge = add_knowledge(knowledge_to_add) + added_knowledge = knowledge_service.add_knowledge(knowledge_to_add) logger.info(f"Knowledge {added_knowledge} added successfully") process_crawl_and_notify.delay( diff --git a/backend/routes/upload_routes.py b/backend/routes/upload_routes.py index cb7276a3ee5e..ff48702960fc 100644 --- a/backend/routes/upload_routes.py +++ b/backend/routes/upload_routes.py @@ -7,7 +7,8 @@ from logger import get_logger from middlewares.auth import AuthBearer, get_current_user from models import UserUsage -from models.databases.supabase.knowledge import CreateKnowledgeProperties +from modules.knowledge.dto.inputs import CreateKnowledgeProperties +from modules.knowledge.service.knowledge_service import KnowledgeService from modules.notification.dto.inputs import ( CreateNotificationProperties, NotificationUpdatableProperties, @@ -17,7 +18,6 @@ from modules.user.entity.user_identity import UserIdentity from packages.files.file import convert_bytes, get_file_size from repository.files.upload_file import upload_file_storage -from repository.knowledge.add_knowledge import add_knowledge from routes.authorizations.brain_authorization import ( RoleEnum, validate_brain_authorization, @@ -27,6 +27,7 @@ upload_router = APIRouter() notification_service = NotificationService() +knowledge_service = KnowledgeService() @upload_router.get("/upload/healthz", tags=["Health"]) @@ -108,7 +109,7 @@ async def upload_file( )[-1].lower(), ) - added_knowledge = add_knowledge(knowledge_to_add) + added_knowledge = knowledge_service.add_knowledge(knowledge_to_add) logger.info(f"Knowledge {added_knowledge} added successfully") process_file_and_notify.delay(