Jim/vectordb update

.github/workflows/deploy.yml

@@ -86,6 +86,14 @@ jobs:
             --var RAG_OPENAI_API_BASE_URL="$RAG_OPENAI_API_BASE_URL"
             --var RAG_OPENAI_API_KEY="$RAG_OPENAI_API_KEY"
             --var RAG_EMBEDDING_ENGINE="$RAG_EMBEDDING_ENGINE"
+      # New Step: Clean up build artifacts
+      - name: Remove build artifacts and commit
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }}  # Use PAT_TOKEN secret for authentication
+        run: |
+          git rm -rf build/
+          git commit -m "Remove build artifacts after deployment" || echo "No changes to commit"
+          git push --force
 
   bail:
     runs-on: ubuntu-latest

.husky/pre-commit

@@ -1,4 +1,4 @@
-echo "Scanning commit history, ~30-60s... 💅🥱⏳"
+echo "Scanning current commit... 💅🥱⏳"
 gitleaks detect --log-opts="-n 1"
 npm install
 npm run format

backend/apps/rag/utils.py

@@ -223,21 +223,50 @@ async def generate_openai_batch_embeddings_async(
 ) -> Optional[list[list[float]]]:
     try:
         if "azure.com" in url:
-            log.info(f"Generating async OpenAI embeddings for {texts[0][0:25]}...")
-            async with session.post(
-                f"{url}/embeddings?api-version=2023-05-15",
-                headers={
-                    "Content-Type": "application/json",
-                    "api-key": f"{key}",
-                },
-                json={"input": texts, "model": model},
-            ) as response:
-                response.raise_for_status()
-                data = await response.json()
-                if "data" in data:
-                    return [elem["embedding"] for elem in data["data"]]
-                else:
-                    raise Exception("Something went wrong :/")
+            initial_delay = 0.2
+            max_retries = 10
+            attempt = 0
+            retry_delay = initial_delay  # Start with the initial delay
+
+            while attempt <= max_retries:
+                log.info(
+                    f"Generating async OpenAI embeddings for {texts[0][0:25]}..."
+                    + f"(Attempt {attempt + 1}/{max_retries + 1})"
+                )
+                async with session.post(
+                    f"{url}/embeddings?api-version=2023-05-15",
+                    headers={
+                        "Content-Type": "application/json",
+                        "api-key": f"{key}",
+                    },
+                    json={"input": texts, "model": model},
+                ) as response:
+                    if response.status == 429:
+                        if attempt < max_retries:
+                            log.warning(
+                                f"Received 429 Too Many Requests. Retrying after {retry_delay:.2f} seconds..."
+                            )
+                            await asyncio.sleep(retry_delay)
+                            attempt += 1
+                            retry_delay *= 2
+                            continue
+                        else:
+                            log.error(
+                                "Max retries exceeded after receiving 429 Too Many Requests."
+                            )
+                            raise Exception(
+                                "429 Too Many Requests: Maximum retries exceeded."
+                            )
+                    elif response.status != 200:
+                        response.raise_for_status()
+
+                    data = await response.json()
+                    if "data" in data:
+                        return [elem["embedding"] for elem in data["data"]]
+                    else:
+                        raise Exception("Something went wrong :/")
+
+            raise Exception("The request failed after several retries.")
         else:
             log.info(
                 f"Generating async Azure OpenAI embeddings for {texts[0][0:25]}..."

backend/config.py

@@ -795,9 +795,12 @@ class BannerModel(BaseModel):
 # Redis connection settings
 REDIS_HOST = os.environ.get("REDIS_HOST", "localhost")
 REDIS_PORT = int(os.environ.get("REDIS_PORT", "6379"))
-REDIS_PASSWORD = os.environ.get(
-    "REDIS_PASSWORD", "you forgot to set the redis password"
-)
+REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", "")
+REDIS_URL = f"redis://localhost:{REDIS_PORT}"
+REDIS_ENABLED = True
+REDIS_SSL = False
+REDIS_SSL_CERT_REQS = None
+REDIS_CONFIG = dict(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD)
 REDIS_DB = int(os.environ.get("REDIS_DB", "0"))
 REDIS_INDEX_NAME = os.environ.get("REDIS_INDEX_NAME", "document-index")
 REDIS_PREFIX = os.environ.get("REDIS_PREFIX", "doc")
@@ -1173,10 +1176,7 @@ class BannerModel(BaseModel):
 # Datastores
 ####################################
 
-REDIS_PORT = 6379
-REDIS_URL = f"redis://localhost:{REDIS_PORT}"
-REDIS_ENABLED = True
-REDIS_CONFIG = dict(host="localhost", port=REDIS_PORT, password="")
+
 DATABASE_URL = os.environ.get("DATABASE_URL", f"sqlite:///{DATA_DIR}/webui.db")
 
 if os.environ.get("VCAP_SERVICES"):
@@ -1198,8 +1198,12 @@ class BannerModel(BaseModel):
         REDIS_CONFIG["password"] = redis_credentials["password"]
         REDIS_CONFIG["ssl"] = True
         REDIS_CONFIG["ssl_cert_reqs"] = None
+        REDIS_SSL = True
+        REDIS_SSL_CERT_REQS = None
+        REDIS_HOST = redis_credentials["host"]
+        REDIS_PORT = int(redis_credentials["port"])
+        REDIS_PASSWORD = redis_credentials["password"]
         REDIS_URL = redis_credentials["uri"].replace("redis://", "rediss://")
-        # REDIS_CONFIG['REDIS_URL'] = REDIS_URL
         REDIS_ENABLED = True
     else:
         REDIS_URL = None
@@ -1213,14 +1217,7 @@ class BannerModel(BaseModel):
 
 
 async def initialize_vector_client():
-    REDIS_CLIENT = Redis(
-        host=REDIS_HOST,
-        port=REDIS_PORT,
-        # password=REDIS_PASSWORD,
-        db=REDIS_DB,
-        decode_responses=False,
-    )
-
+    REDIS_CLIENT = Redis(**REDIS_CONFIG)
     return REDIS_CLIENT