Implement Filecatcher

app/chainlit_app.py

@@ -3,9 +3,8 @@
 import sys, os, yaml, torch
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../')))
 
-from src.components import RagPipeline, load_config, AudioTranscriber
+from src.components import RagPipeline, load_config
 from loguru import logger
-from io import BytesIO
 
 APP_DIR = Path(__file__).parent.absolute() # Path.cwd().parent.absolute()
 UPLOAD_DIR = APP_DIR / "upload_dir"

main.py

@@ -2,7 +2,8 @@
 import asyncio
 from loguru import logger
 from pathlib import Path
-from src.components import RagPipeline, load_config, evaluate, Indexer
+from src.components import RagPipeline, load_config, evaluate
+from filecatcher.components import Indexer
 
 # config_path = Path(__file__) / '.hydra_config'
 config = load_config()

manage_collection.py

@@ -47,12 +47,12 @@ async def main():
     print(config)
 
     if args.folder:
-        from src.components import Indexer
+        from filecatcher.components import Indexer
 
         collection = config.vectordb["collection_name"]
         logger.warning(f"Data will be upserted to the collection {collection}")
 
-        indexer = Indexer(config, logger)
+        indexer = Indexer(config = config, logger = logger)
 
         start = time.time()
         await indexer.add_files2vdb(path=args.folder)

pyproject.toml

@@ -17,6 +17,7 @@ langchain-community = "^0.3.7"
 langchain-openai = "^0.2.8"
 ragatouille = "^0.0.8.post4"
 whisperx = {git = "https://github.com/federicotorrielli/BetterWhisperX", branch="main"}
+filecatcher = {git = "https://github.com/OpenLLM-France/fileCatcher.git"}
 pyannote-audio = "3.1.1"
 ctranslate2 = "4.4.0"
 qdrant-client = "^1.12.1"

src/components/config.py

@@ -1,19 +1,9 @@
-import os
-from dotenv import load_dotenv, find_dotenv
 from omegaconf import OmegaConf
 from pathlib import Path
 from hydra import initialize, compose
 
-def load_config(config_path="../../.hydra_config", overrides=None)-> OmegaConf:
-    load_dotenv()
-    print(overrides)
-
+def load_config(config_path="../../.hydra_config", overrides=None)-> OmegaConf:  
     with initialize(config_path=config_path, job_name="config_loader"):
         config = compose(config_name="config", overrides=overrides)
         config.paths.root_dir = Path(config.paths.root_dir).absolute()
-        return config
-
-# # Example usage
-# if __name__ == "__main__":
-#     config = load_config()
-#     print(config)
+        return config