Merge pull request #14 from sofyan48/elastic-vectorstore

Add: Support Elastic vectorestore

app/__init__.py

@@ -16,7 +16,8 @@
     minio,
     jwt,
     deepseek,
-    ollama
+    ollama,
+    elasticsearch
 )
 from starlette.middleware.cors import CORSMiddleware
 
@@ -30,6 +31,8 @@
 register_alchemy_async()
 prompter.regist_default_prompter()
 minio.register_storage_minio()
+elasticsearch.register_elasticsearch()
+vectorstore.register_elasticsearch_vectorstore()
 jwt.register_jwt()
 
 

app/library/__init__.py

@@ -9,14 +9,19 @@
 from pkg.storage.minio import StorageMinio
 from pkg.deepseek import DeepSeekLLM
 from pkg.ollama import OllamaPlatform
+from pkg.vectorstore.elasticsearch import ElasticsearcVector
+from app.library.vectorstore import Vectorstores
 
 mistral_llm = MistralLLM()
 openai_llm = OpenAILLM()
 groq_llm = GroqLLM()
 deepseek_llm = DeepSeekLLM()
 ollama_llm = OllamaPlatform()
 chromadb = ChromaDB()
+elastic = ElasticsearcVector
 minio = StorageMinio
+vectorstoreDB = Vectorstores()
+
 
 from app.repositories.setup import SetupConfig
 from app.repositories.prompt import PromptRepositories

app/library/deepseek.py

@@ -1,26 +1,28 @@
 from pkg.deepseek import DeepSeekLLM, Runnable
-from pkg.vectorstore.chromadb import ChromaDB, VectorStoreRetriever
+from pkg.vectorstore.chromadb import VectorStoreRetriever
 from app import redis
+from app.library.vectorstore import Vectorstores
 from pkg.chain import Chain, RunnableWithMessageHistory
 from pkg.history import RedisChatMessageHistory, SQLChatMessageHistory
 from pkg.chain.prompter import PromptTemplate
 
 class DeepSeekLibrary(object):
-    def __init__(self, chroma: ChromaDB, llm: DeepSeekLLM, model: str, redis: redis):
-        self.chroma = chroma
+    def __init__(self, vectorstores: Vectorstores, llm: DeepSeekLLM, model: str, redis: redis):
+        self.vectorstore = vectorstores
         self.deepseek = llm
         self.model = model
         self.redis = redis
         self.chain = Chain()
 
-    def retriever(self, top_k, fetch_k, collection) -> VectorStoreRetriever:
+    def retriever(self, vector, top_k, fetch_k, collection) -> VectorStoreRetriever:
         if top_k is None:
             top_k = 3
 
         if fetch_k is None:
             fetch_k = 10
         try:
-            return self.chroma.retriever(
+            vectordb = self.vectorstore.configure(vector)
+            return vectordb.chroma.retriever(
                 topK=top_k,
                 fetchK=fetch_k,
                 collection=collection

app/library/groq.py

@@ -1,25 +1,27 @@
 from pkg.groq import GroqLLM, Runnable
-from pkg.vectorstore.chromadb import ChromaDB, VectorStoreRetriever
+from pkg.vectorstore.chromadb import VectorStoreRetriever
+from app.library.vectorstore import Vectorstores
 from app import redis
 from pkg.chain import Chain, RunnableWithMessageHistory
 from pkg.history import RedisChatMessageHistory
 from pkg.chain.prompter import PromptTemplate
 
 class GroqAILibrary(object):
-    def __init__(self, chroma: ChromaDB, llm: GroqLLM, model: str, redis: redis):
-        self.chroma = chroma
+    def __init__(self, vectorstores: Vectorstores, llm: GroqLLM, model: str, redis: redis):
+        self.vectorstore = vectorstores
         self.groq = llm
         self.model = model
         self.redis = redis
         self.chain = Chain()
 
-    def retriever(self, top_k, fetch_k, collection) -> VectorStoreRetriever:
+    def retriever(self, vector: str, top_k, fetch_k, collection) -> VectorStoreRetriever:
         if top_k is None:
             top_k = 3
         if fetch_k is None:
             fetch_k = 10
         try:
-            return self.chroma.retriever(
+            vectordb = self.vectorstore.configure(vector)
+            return vectordb.retriever(
                 topK=top_k,
                 fetchK=fetch_k,
                 collection=collection

app/library/mistral.py

@@ -1,25 +1,27 @@
 from pkg.mistral import MistralLLM, Runnable
-from pkg.vectorstore.chromadb import ChromaDB, VectorStoreRetriever
+from pkg.vectorstore.chromadb import VectorStoreRetriever
+from app.library.vectorstore import Vectorstores
 from app import redis
 from pkg.chain import Chain, RunnableWithMessageHistory
 from pkg.history import RedisChatMessageHistory
 from pkg.chain.prompter import PromptTemplate
 
 class MistralAILibrary(object):
-    def __init__(self, chroma: ChromaDB, llm: MistralLLM, model: str, redis: redis):
-        self.chroma = chroma
+    def __init__(self, vectorstores: Vectorstores, llm: MistralLLM, model: str, redis: redis):
+        self.vectorstore = vectorstores
         self.mistral = llm
         self.redis = redis
         self.model = model
         self.chain = Chain()
 
-    def retriever(self, top_k, fetch_k, collection) -> VectorStoreRetriever:
+    def retriever(self, vector, top_k, fetch_k, collection) -> VectorStoreRetriever:
         if top_k is None:
             top_k = 3
         if fetch_k is None:
             fetch_k = 10
         try:
-            return self.chroma.retriever(
+            vectordb = self.vectorstore.configure(vector)
+            return vectordb.retriever(
                 topK=top_k,
                 fetchK=fetch_k,
                 collection=collection

app/library/ollama.py

@@ -1,13 +1,14 @@
 from pkg.ollama import OllamaPlatform, Runnable
-from pkg.vectorstore.chromadb import ChromaDB, VectorStoreRetriever
+from pkg.vectorstore.chromadb import VectorStoreRetriever
 from app import redis
+from app.library.vectorstore import Vectorstores
 from pkg.chain import Chain, RunnableWithMessageHistory
 from pkg.history import RedisChatMessageHistory, SQLChatMessageHistory
 from pkg.chain.prompter import PromptTemplate
 
 class OllamaLibrary(object):
-    def __init__(self, chroma: ChromaDB, llm: OllamaPlatform, model: str, redis: redis, base_url=None, top_k=4, top_p=0.2):
-        self.chroma = chroma
+    def __init__(self, vectorstores: Vectorstores, llm: OllamaPlatform, model: str, redis: redis, base_url=None, top_k=4, top_p=0.2):
+        self.vectorstores = vectorstores
         self.ollama = llm
         self.model = model
         self.redis = redis
@@ -16,14 +17,15 @@ def __init__(self, chroma: ChromaDB, llm: OllamaPlatform, model: str, redis: red
         self.top_k = top_k
         self.base_url = base_url
 
-    def retriever(self, top_k, fetch_k, collection) -> VectorStoreRetriever:
+    def retriever(self, vector, top_k, fetch_k, collection) -> VectorStoreRetriever:
         if top_k is None:
             top_k = 3
 
         if fetch_k is None:
             fetch_k = 10
         try:
-            return self.chroma.retriever(
+            vectordb = self.vectorstore.configure(vector)
+            return vectordb.retriever(
                 topK=top_k,
                 fetchK=fetch_k,
                 collection=collection

app/library/openai.py

@@ -1,26 +1,28 @@
 from pkg.openai import OpenAILLM, Runnable
-from pkg.vectorstore.chromadb import ChromaDB, VectorStoreRetriever
+from pkg.vectorstore.chromadb import VectorStoreRetriever
+from app.library.vectorstore import Vectorstores
 from app import redis
 from pkg.chain import Chain, RunnableWithMessageHistory
 from pkg.history import RedisChatMessageHistory, SQLChatMessageHistory
 from pkg.chain.prompter import PromptTemplate
 
 class OpenAILibrary(object):
-    def __init__(self, chroma: ChromaDB, llm: OpenAILLM, model: str, redis: redis):
-        self.chroma = chroma
+    def __init__(self, vectorstores: Vectorstores, llm: OpenAILLM, model: str, redis: redis):
+        self.vectorstore = vectorstores
         self.openai = llm
         self.model = model
         self.redis = redis
         self.chain = Chain()
 
-    def retriever(self, top_k, fetch_k, collection) -> VectorStoreRetriever:
+    def retriever(self, vector: str, top_k, fetch_k, collection) -> VectorStoreRetriever:
         if top_k is None:
             top_k = 3
 
         if fetch_k is None:
             fetch_k = 10
         try:
-            return self.chroma.retriever(
+            vectordb = self.vectorstore.configure(vector)
+            return vectordb.retriever(
                 topK=top_k,
                 fetchK=fetch_k,
                 collection=collection

app/library/setup.py

@@ -30,13 +30,13 @@ async def save_all(cls):
             try:
                 await cls._instance_redis.set(i.get('key'), i.get('value'))
             except Exception as e:
-                raise e
+                logger.warning("No config found using default configuration")
 
         try:
             data_prompt = await cls._instance_repo_prompt.get_prompt_config()
             await cls._instance_redis.set("config:prompt", data_prompt.prompt)
         except Exception as e:
-            raise e
+            logger.warning("No prompt database found using default configuration")
 
     @classmethod
     async def get_all_config(cls):

app/library/vectorstore.py

@@ -0,0 +1,16 @@
+from pkg.vectorstore.chromadb import ChromaDB
+from pkg.vectorstore.elasticsearch import ElasticsearcVector
+
+
+class Vectorstores(object):
+    def __init__(self):
+        pass
+
+    def configure(self, vectorestore = None) -> ChromaDB | ElasticsearcVector:
+        if vectorestore is None:
+            return ChromaDB
+
+        if vectorestore=="elastic":
+            return ElasticsearcVector
+        else:
+            return ChromaDB

app/library/wrapper.py

@@ -12,7 +12,7 @@
     mistral_llm, 
     openai_llm, 
     groq_llm, 
-    chromadb,
+    vectorstoreDB,
     deepseek_llm,
     ollama_llm
 )
@@ -37,7 +37,7 @@ def mistral(self, model=None):
         if model is None:
             model = "open-mistral-nemo"
         return MistralAILibrary(
-            chroma=chromadb,
+            vectorstores=vectorstoreDB,
             llm=mistral_llm,
             redis=redis,
             model=model
@@ -46,8 +46,9 @@ def mistral(self, model=None):
     def openai(self, model=None):
         if model is None:
             model = "gpt-4o-mini"
+
         return OpenAILibrary(
-            chroma=chromadb,
+            vectorstores=vectorstoreDB,
             llm=openai_llm,
             redis=redis,
             model=model
@@ -57,7 +58,7 @@ def groq(self, model="llama-3.3-70b-versatile"):
         if model is None:
             model = "llama-3.3-70b-versatile"
         return GroqAILibrary(
-            chroma=chromadb,
+            vectorstores=vectorstoreDB,
             llm=groq_llm,
             redis=redis,
             model=model
@@ -67,7 +68,7 @@ def deepseek(self, model=None):
         if model is None:
             model = "deepseek-chat"
         return DeepSeekLibrary(
-            chroma=chromadb,
+            vectorstores=vectorstoreDB,
             llm=deepseek_llm,
             redis=redis,
             model=model
@@ -77,13 +78,13 @@ def ollama(self, model=None):
         if model is None:
             model = "deepseek-r1:1.5b"
         return OllamaLibrary(
-            chroma=chromadb,
+            vectorstores=vectorstoreDB,
             llm=ollama_llm,
             redis=redis,
             model=model,
         )
 
-    def initiate(self, llm: str = "mistral", model=None) -> (OpenAILibrary | MistralAILibrary | GroqAILibrary| DeepSeekLibrary):
+    def initiate(self, llm: str = "mistral", model=None) -> (OpenAILibrary | MistralAILibrary | GroqAILibrary| DeepSeekLibrary| OllamaLibrary):
         if model is None:
             model = None
         if llm == "mistral":

app/presentation/request.py

@@ -53,6 +53,6 @@ class RequestGrantClient(BaseModel):
     api_key: str = Field(..., min_length=1)
     secret_key: str = Field(..., min_length=1)
 
-class RequestIngestChroma(BaseModel):
+class RequestIngestVector(BaseModel):
     collection: str = Field(..., min_length=1)
     ingest_code: str = Field(..., min_length=1)

app/router/http.py

@@ -14,11 +14,11 @@
 
 
 from app.ucase.ingest import (
-    chroma,
+    build,
     ingest_upload,
     ingest_list
 )
-routerV1.include_router(chroma.router)
+routerV1.include_router(build.router)
 routerV1.include_router(ingest_upload.router)
 routerV1.include_router(ingest_list.router)
 

app/ucase/chat/chat.py

@@ -62,8 +62,14 @@ async def send_chat(
                 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
                 detail="Please setup retriever collection or set from payload"
             )
+    try:
+        vectorDB = int(setup.get('config:retriever:vector_db'))
+    except Exception:
+        vectorDB = "chroma"
+
     try:
         retriever = llm.retriever(
+            vector=vectorDB,
             top_k=top_k,
             fetch_k=fetch_k,
             collection=collection

app/ucase/ingest/__init__.py

@@ -8,13 +8,15 @@
 from pkg.vectorstore.chromadb import ChromaDB
 from app.repositories import setup
 from app.repositories.ingest_document import IngestDocumentRepositories
+from app.repositories import setup
+from app.library import vectorstoreDB
 
 
 auth = BearerAuthentication()
 router = APIRouter()
 setup_repo = setup.SetupConfig()
 chromadb = ChromaDB()
-
+setup_repo = setup.SetupConfig()
 ingest_docs_repo = IngestDocumentRepositories()