refactor: knowledge module (#1743)

# Description

- Refactor knowledge to a module

- This PR breaks the Github Processor function -> need to comment brain
and file imports as it creates a circular dependency issue. Should be
fixed and reverted in next PR.

.vscode/settings.json

@@ -7,6 +7,7 @@
   },
   "python.linting.enabled": true,
   "python.linting.flake8Enabled": true,
+  "python.analysis.extraPaths": ["./backend"],
   "editor.formatOnSave": true,
   "[python]": {
     "editor.defaultFormatter": "ms-python.black-formatter",

backend/main.py

@@ -13,6 +13,7 @@
 from fastapi.responses import JSONResponse
 from logger import get_logger
 from middlewares.cors import add_cors_middleware
+from modules.knowledge.controller.knowledge_routes import knowledge_router
 from modules.notification.controller.notification_routes import notification_router
 from modules.onboarding.controller.onboarding_routes import onboarding_router
 from modules.prompt.controller.prompt_routes import prompt_router
@@ -22,7 +23,6 @@
 from routes.chat_routes import chat_router
 from routes.contact_routes import router as contact_router
 from routes.crawl_routes import crawl_router
-from routes.knowledge_routes import knowledge_router
 from routes.misc_routes import misc_router
 from routes.subscription_routes import subscription_router
 from routes.upload_routes import upload_router

backend/models/databases/repository.py

@@ -204,26 +204,6 @@ def get_vectors_in_batch(self, batch_ids):
     def get_vectors_by_file_sha1(self, file_sha1):
         pass
 
-    @abstractmethod
-    def insert_knowledge(self, brain_id: UUID):
-        pass
-
-    @abstractmethod
-    def remove_knowledge_by_id(self, knowledge_id: UUID):
-        pass
-
-    @abstractmethod
-    def remove_brain_all_knowledge(self, brain_id: UUID):
-        pass
-
-    @abstractmethod
-    def get_knowledge_by_id(self, knowledge_id: UUID):
-        pass
-
-    @abstractmethod
-    def get_all_knowledge_in_brain(self, brain_id: UUID):
-        pass
-
     @abstractmethod
     def get_api_brain_definition(self, brain_id: UUID):
         pass

backend/models/databases/supabase/__init__.py

@@ -5,6 +5,5 @@
     BrainSubscription
 from models.databases.supabase.chats import Chats
 from models.databases.supabase.files import File
-from models.databases.supabase.knowledge import Knowledges
 from models.databases.supabase.user_usage import UserUsage
 from models.databases.supabase.vectors import Vector

backend/models/databases/supabase/supabase.py

@@ -6,7 +6,6 @@
     BrainSubscription,
     Chats,
     File,
-    Knowledges,
     UserUsage,
     Vector,
 )
@@ -22,7 +21,6 @@ class SupabaseDB(
     ApiKeyHandler,
     Chats,
     Vector,
-    Knowledges,
     ApiBrainDefinitions,
 ):
     def __init__(self, supabase_client):
@@ -34,5 +32,4 @@ def __init__(self, supabase_client):
         ApiKeyHandler.__init__(self, supabase_client)
         Chats.__init__(self, supabase_client)
         Vector.__init__(self, supabase_client)
-        Knowledges.__init__(self, supabase_client)
         ApiBrainDefinitions.__init__(self, supabase_client)

backend/modules/knowledge/controller/__init__.py

@@ -0,0 +1 @@
+from .knowledge_routes import knowledge_router

backend/routes/knowledge_routes.py → backend/modules/knowledge/controller/knowledge_routes.py

@@ -4,13 +4,10 @@
 from logger import get_logger
 from middlewares.auth import AuthBearer, get_current_user
 from models import Brain
+from modules.knowledge.service.knowledge_service import KnowledgeService
 from modules.user.entity.user_identity import UserIdentity
 from repository.files.delete_file import delete_file_from_storage
 from repository.files.generate_file_signed_url import generate_file_signed_url
-from repository.knowledge.get_all_knowledge import get_all_knowledge
-from repository.knowledge.get_knowledge import get_knowledge
-from repository.knowledge.remove_knowledge import remove_knowledge
-
 from routes.authorizations.brain_authorization import (
     RoleEnum,
     has_brain_authorization,
@@ -20,6 +17,8 @@
 knowledge_router = APIRouter()
 logger = get_logger(__name__)
 
+knowledge_service = KnowledgeService()
+
 
 @knowledge_router.get(
     "/knowledge", dependencies=[Depends(AuthBearer())], tags=["Knowledge"]
@@ -34,7 +33,7 @@ async def list_knowledge_in_brain_endpoint(
 
     validate_brain_authorization(brain_id=brain_id, user_id=current_user.id)
 
-    knowledges = get_all_knowledge(brain_id)
+    knowledges = knowledge_service.get_all_knowledge(brain_id)
     logger.info(f"List of knowledge from knowledge table: {knowledges}")
 
     return {"knowledges": knowledges}
@@ -59,9 +58,9 @@ async def delete_endpoint(
 
     brain = Brain(id=brain_id)
 
-    knowledge = get_knowledge(knowledge_id)
+    knowledge = knowledge_service.get_knowledge(knowledge_id)
     file_name = knowledge.file_name if knowledge.file_name else knowledge.url
-    remove_knowledge(knowledge_id)
+    knowledge_service.remove_knowledge(knowledge_id)
 
     if knowledge.file_name:
         delete_file_from_storage(f"{brain_id}/{knowledge.file_name}")
@@ -87,7 +86,7 @@ async def generate_signed_url_endpoint(
     Generate a signed url to download the file from storage.
     """
 
-    knowledge = get_knowledge(knowledge_id)
+    knowledge = knowledge_service.get_knowledge(knowledge_id)
 
     validate_brain_authorization(brain_id=knowledge.brain_id, user_id=current_user.id)
 

backend/modules/knowledge/dto/__init__.py

@@ -0,0 +1,2 @@
+from .inputs import CreateKnowledgeProperties
+from .outputs import DeleteKnowledgeResponse

backend/modules/knowledge/dto/inputs.py

@@ -0,0 +1,16 @@
+from typing import Optional
+from uuid import UUID
+
+from pydantic import BaseModel
+
+
+class CreateKnowledgeProperties(BaseModel):
+    brain_id: UUID
+    file_name: Optional[str] = None
+    url: Optional[str] = None
+    extension: str = "txt"
+
+    def dict(self, *args, **kwargs):
+        knowledge_dict = super().dict(*args, **kwargs)
+        knowledge_dict["brain_id"] = str(knowledge_dict.get("brain_id"))
+        return knowledge_dict

backend/modules/knowledge/dto/outputs.py

@@ -0,0 +1,8 @@
+from uuid import UUID
+
+from pydantic import BaseModel
+
+
+class DeleteKnowledgeResponse(BaseModel):
+    status: str = "delete"
+    knowledge_id: UUID

backend/modules/knowledge/entity/__init__.py

@@ -0,0 +1 @@
+from .knowledge import Knowledge

backend/modules/knowledge/repository/__init__.py

@@ -0,0 +1 @@
+from .knowledges import Knowledges

backend/modules/knowledge/repository/knowledge_interface.py

@@ -0,0 +1,58 @@
+from abc import ABC, abstractmethod
+from typing import List
+from uuid import UUID
+
+from modules.knowledge.dto.inputs import CreateKnowledgeProperties
+from modules.knowledge.dto.outputs import DeleteKnowledgeResponse
+from modules.knowledge.entity.knowledge import Knowledge
+
+
+class KnowledgeInterface(ABC):
+    @abstractmethod
+    def insert_knowledge(self, knowledge: CreateKnowledgeProperties) -> Knowledge:
+        """
+        Add a knowledge
+        """
+        pass
+
+    @abstractmethod
+    def remove_knowledge_by_id(
+        # todo: update remove brain endpoints to first delete the knowledge
+        self,
+        knowledge_id: UUID,
+    ) -> DeleteKnowledgeResponse:
+        """
+        Args:
+            knowledge_id (UUID): The id of the knowledge
+
+        Returns:
+            str: Status message
+        """
+        pass
+
+    @abstractmethod
+    def get_knowledge_by_id(self, knowledge_id: UUID) -> Knowledge:
+        """
+        Get a knowledge by its id
+        Args:
+            brain_id (UUID): The id of the brain
+        """
+        pass
+
+    @abstractmethod
+    def get_all_knowledge_in_brain(self, brain_id: UUID) -> List[Knowledge]:
+        """
+        Get all the knowledge in a brain
+        Args:
+            brain_id (UUID): The id of the brain
+        """
+        pass
+
+    @abstractmethod
+    def remove_brain_all_knowledge(self, brain_id: UUID) -> None:
+        """
+        Remove all knowledge in a brain
+        Args:
+            brain_id (UUID): The id of the brain
+        """
+        pass

backend/models/databases/supabase/knowledge.py → backend/modules/knowledge/repository/knowledges.py

@@ -1,34 +1,16 @@
-from typing import List, Optional
-from uuid import UUID
-
 from fastapi import HTTPException
-from models.databases.repository import Repository
-from models.knowledge import Knowledge
-from pydantic import BaseModel
-
-
-class CreateKnowledgeProperties(BaseModel):
-    brain_id: UUID
-    file_name: Optional[str] = None
-    url: Optional[str] = None
-    extension: str = "txt"
-
-    def dict(self, *args, **kwargs):
-        knowledge_dict = super().dict(*args, **kwargs)
-        knowledge_dict["brain_id"] = str(knowledge_dict.get("brain_id"))
-        return knowledge_dict
-
-
-class DeleteKnowledgeResponse(BaseModel):
-    status: str = "delete"
-    knowledge_id: UUID
+from models.settings import get_supabase_client
+from modules.knowledge.dto.outputs import DeleteKnowledgeResponse
+from modules.knowledge.entity.knowledge import Knowledge
+from modules.knowledge.repository.knowledge_interface import KnowledgeInterface
 
 
-class Knowledges(Repository):
-    def __init__(self, supabase_client):
+class Knowledges(KnowledgeInterface):
+    def __init__(self):
+        supabase_client = get_supabase_client()
         self.db = supabase_client
 
-    def insert_knowledge(self, knowledge: CreateKnowledgeProperties) -> Knowledge:
+    def insert_knowledge(self, knowledge):
         """
         Add a knowledge
         """
@@ -38,8 +20,8 @@ def insert_knowledge(self, knowledge: CreateKnowledgeProperties) -> Knowledge:
     def remove_knowledge_by_id(
         # todo: update remove brain endpoints to first delete the knowledge
         self,
-        knowledge_id: UUID,
-    ) -> DeleteKnowledgeResponse:
+        knowledge_id,
+    ):
         """
         Args:
             knowledge_id (UUID): The id of the knowledge
@@ -64,7 +46,7 @@ def remove_knowledge_by_id(
             knowledge_id=knowledge_id,
         )
 
-    def get_knowledge_by_id(self, knowledge_id: UUID) -> Knowledge:
+    def get_knowledge_by_id(self, knowledge_id):
         """
         Get a knowledge by its id
         Args:
@@ -79,7 +61,7 @@ def get_knowledge_by_id(self, knowledge_id: UUID) -> Knowledge:
 
         return Knowledge(**knowledge[0])
 
-    def get_all_knowledge_in_brain(self, brain_id: UUID) -> List[Knowledge]:
+    def get_all_knowledge_in_brain(self, brain_id):
         """
         Get all the knowledge in a brain
         Args:
@@ -94,7 +76,7 @@ def get_all_knowledge_in_brain(self, brain_id: UUID) -> List[Knowledge]:
 
         return [Knowledge(**knowledge) for knowledge in all_knowledge]
 
-    def remove_brain_all_knowledge(self, brain_id: UUID) -> None:
+    def remove_brain_all_knowledge(self, brain_id):
         """
         Remove all knowledge in a brain
         Args:

backend/modules/knowledge/service/knowledge_service.py

@@ -0,0 +1,46 @@
+from uuid import UUID
+
+from logger import get_logger
+from modules.knowledge.dto.inputs import CreateKnowledgeProperties
+from modules.knowledge.entity.knowledge import Knowledge
+from modules.knowledge.repository.knowledge_interface import KnowledgeInterface
+from modules.knowledge.repository.knowledges import Knowledges
+
+logger = get_logger(__name__)
+
+
+class KnowledgeService:
+    repository: KnowledgeInterface
+
+    def __init__(self):
+        self.repository = Knowledges()
+
+    def add_knowledge(self, knowledge_to_add: CreateKnowledgeProperties):
+        knowledge = self.repository.insert_knowledge(knowledge_to_add)
+
+        logger.info(f"Knowledge { knowledge.id} added successfully")
+        return knowledge
+
+    def get_all_knowledge(self, brain_id: UUID):
+        knowledges = self.repository.get_all_knowledge_in_brain(brain_id)
+
+        return knowledges
+
+    def get_knowledge(self, knowledge_id: UUID) -> Knowledge:
+        knowledge = self.repository.get_knowledge_by_id(knowledge_id)
+
+        return knowledge
+
+    def remove_brain_all_knowledge(self, brain_id: UUID) -> None:
+        self.repository.remove_brain_all_knowledge(brain_id)
+
+        logger.info(
+            f"All knowledge in brain {brain_id} removed successfully from table"
+        )
+
+    def remove_knowledge(self, knowledge_id: UUID):
+        message = self.repository.remove_knowledge_by_id(knowledge_id)
+
+        logger.info(f"Knowledge { knowledge_id} removed successfully from table")
+
+        return message