Merge pull request #16 from MirageML/fix/sree/loop-through-data

Fix/sree/loop through data

.github/workflows/check-lint.yml

@@ -28,22 +28,3 @@ jobs:
           version: "3.10"
 
       - run: inv lint
-
-  build-version:
-    name: Build version
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Python
-        uses: ./.github/actions/setup-cached-python
-        with:
-          version: "3.9"
-
-      - name: Bump the version number
-        run: inv update-build-number
-
-      - uses: stefanzweifel/git-auto-commit-action@v5
-        with:
-          commit_message: Bump the build number

.github/workflows/update-version.yml

@@ -0,0 +1,25 @@
+name: Build Version Once Per PR
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+
+jobs:
+  build-version:
+    name: Build version
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: ./.github/actions/setup-cached-python
+        with:
+          version: "3.9"
+
+      - name: Bump the version number
+        run: inv update-build-number
+
+      - uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: Bump the build number

mirageml/__main__.py

@@ -158,16 +158,17 @@ def add_plugin_command(name: str):
 @add_app.command(name="source")
 def add_source_command():
     """Add a new source"""
-    from .commands import add_source
-
     while True:
         link = input("Link for the source: ")
         if not link.startswith("https://"):
             typer.echo("Please enter a valid link starting with https://")
             continue
         break
+
     from urllib.parse import urlparse
 
+    from .commands import add_source
+
     parsed_url = urlparse(link)
     name = parsed_url.netloc.split(".")[0]
     if name == "docs":

mirageml/commands/list_sources.py

@@ -9,17 +9,29 @@ def set_sources():
 
 
 def list_sources():
+    import sys
+
     from .config import set_var_config
     from .utils.vectordb import list_qdrant_db, list_remote_qdrant_db
 
     sources = list_qdrant_db()
+    remote_sources = list_remote_qdrant_db()
+
+    if len(sources) == 0 and len(remote_sources) == 0:
+        typer.secho(
+            f"No sources found. Please add a source using {sys.argv[0].split('/')[-1]} add source",
+            fg=typer.colors.RED,
+            bold=True,
+        )
+        return
 
     if len(sources) != 0:
         typer.secho("Local Sources:", fg=typer.colors.BRIGHT_GREEN, bold=True)
         print("* " + "\n* ".join(sources))
-    remote_sources = list_remote_qdrant_db()
-    print("------------------")
+
     if len(remote_sources) != 0:
+        if len(sources) != 0:
+            print("------------------")
         typer.secho("Remote Sources:", fg=typer.colors.BRIGHT_GREEN, bold=True)
         print("* " + "\n* ".join(remote_sources))
 

mirageml/commands/utils/vectordb.py

@@ -17,6 +17,7 @@
     VECTORDB_DELETE_ENDPOINT,
     VECTORDB_LIST_ENDPOINT,
     VECTORDB_SEARCH_ENDPOINT,
+    VECTORDB_UPSERT_ENDPOINT,
     get_headers,
 )
 from ..config import load_config
@@ -40,12 +41,7 @@ def exists_qdrant_db(collection_name="test"):
 
 
 def create_remote_qdrant_db(data, metadata, collection_name="test"):
-    json_data = {
-        "user_id": keyring.get_password(SERVICE_ID, "user_id"),
-        "data": data,
-        "metadata": metadata,
-        "collection_name": collection_name,
-    }
+    user_id = keyring.get_password(SERVICE_ID, "user_id")
 
     from rich.console import Console
     from rich.live import Live
@@ -63,17 +59,27 @@ def create_remote_qdrant_db(data, metadata, collection_name="test"):
         auto_refresh=True,
         vertical_overflow="visible",
     ) as live:
-        response = requests.post(VECTORDB_CREATE_ENDPOINT, json=json_data, headers=get_headers(), stream=True)
-        if response.status_code == 200:
-            for chunk in response.iter_lines():
-                # process line here
-                live.update(
-                    Panel(
-                        f"Indexing: {chunk.decode()}",
-                        title="[bold green]Indexer[/bold green]",
-                        border_style="green",
+        for i, curr_data in enumerate(data):
+            json_data = {
+                "user_id": user_id,
+                "collection_name": collection_name,
+                "data": [curr_data],
+                "metadata": [metadata[i]],
+            }
+            if i == 0:
+                response = requests.post(VECTORDB_CREATE_ENDPOINT, json=json_data, headers=get_headers(), stream=True)
+            else:
+                response = requests.post(VECTORDB_UPSERT_ENDPOINT, json=json_data, headers=get_headers(), stream=True)
+            if response.status_code == 200:
+                for chunk in response.iter_lines():
+                    # process line here
+                    live.update(
+                        Panel(
+                            f"Indexing: {chunk.decode()}",
+                            title="[bold green]Indexer[/bold green]",
+                            border_style="green",
+                        )
                     )
-                )
 
     typer.secho(f"Created Source: {collection_name}", fg=typer.colors.GREEN, bold=True)
     set_sources()

mirageml/constants.py

@@ -23,6 +23,7 @@
 VECTORDB_SEARCH_ENDPOINT = "https://mirageml--vectordb-search-db.modal.run"
 VECTORDB_LIST_ENDPOINT = "https://mirageml--vectordb-list-db.modal.run"
 VECTORDB_CREATE_ENDPOINT = "https://mirageml--vectordb-create-db.modal.run"
+VECTORDB_UPSERT_ENDPOINT = "https://mirageml--vectordb-upsert-db.modal.run"
 VECTORDB_DELETE_ENDPOINT = "https://mirageml--vectordb-delete-db.modal.run"
 
 supabase = create_client(SUPABASE_URL, SUPABASE_KEY)

mirageml_version/_version_generated.py

@@ -1,2 +1,2 @@
 # Copyright Mirage ML 2023
-build_number = 5
+build_number = 8