Add streamlit demo to simple-rag template

superduper-io · Dec 3, 2024 · a63a857 · a63a857
1 parent 7cde03f
commit a63a857
Show file tree

Hide file tree

Showing 11 changed files with 132 additions and 110 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,4 @@
-# superduper.io Changelog
+# superduper.io Changelog 
 
 All notable changes to this project will be documented in this file.
 

diff --git a/superduper/components/streamlit.py b/superduper/components/streamlit.py
@@ -25,6 +25,8 @@ def page(self):
         def demo_func():
             return self.demo_func(db=self.db, **self.demo_kwargs)
 
+        demo_func.__name__ = self.identifier
+
         return st.Page(demo_func, title=self.identifier, default=self.default)
 
 
@@ -48,7 +50,7 @@ def func_default():
 
     import streamlit as st
 
-    landing = st.Page(func_default, title="Landing", default=True)
+    landing = st.Page(func_default, title="About", default=True)
     pg = st.navigation([landing, *pages])
-    st.set_page_config(page_title="Data manager", page_icon=":material/edit:")
+    st.set_page_config(page_title="Superduper demo server", page_icon=":material/edit:")
     pg.run()
diff --git a/.../dbea2bb8f7b2c787b1043be4588887ff295657b5 → .../1e97b8191f4d7e8d29ae44e247d91f1b4c015614 b/.../dbea2bb8f7b2c787b1043be4588887ff295657b5 → .../1e97b8191f4d7e8d29ae44e247d91f1b4c015614
diff --git a/templates/simple_rag/blobs/8fcce33acf9802bdceac19fa0b5a30d30af40502 b/templates/simple_rag/blobs/8fcce33acf9802bdceac19fa0b5a30d30af40502
diff --git a/.../d7158fe40ed9247bc5a782440a0583977bdf869b → .../a8139b9444e9c9d46f4a6da63a26f9db7af85f8e b/.../d7158fe40ed9247bc5a782440a0583977bdf869b → .../a8139b9444e9c9d46f4a6da63a26f9db7af85f8e
diff --git a/templates/simple_rag/blobs/c663645459821c0f6a085c0f21d2ae498d037bdd b/templates/simple_rag/blobs/c663645459821c0f6a085c0f21d2ae498d037bdd
diff --git a/templates/simple_rag/blobs/ee6badd983a68b245bf5824ae653b807ab349fb4 b/templates/simple_rag/blobs/ee6badd983a68b245bf5824ae653b807ab349fb4
diff --git a/templates/simple_rag/build.ipynb b/templates/simple_rag/build.ipynb
@@ -79,16 +79,14 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "\u001b[32m2024-Dec-03 10:16:47.82\u001b[0m| \u001b[1mINFO    \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.misc.plugins\u001b[0m:\u001b[36m13  \u001b[0m | \u001b[1mLoading plugin: ibis\u001b[0m\n",
-      "\u001b[32m2024-Dec-03 10:16:48.16\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper_ibis.data_backend\u001b[0m:\u001b[36m112 \u001b[0m | \u001b[33m\u001b[1mUnable to connect to the database with self.conn.con: <sqlite3.Connection object at 0x1230cfd40> and self.name: sqlite:. Error: 'sqlite3.Connection' object has no attribute 'dialect'.\u001b[0m\n",
-      "\u001b[32m2024-Dec-03 10:16:48.16\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper_ibis.data_backend\u001b[0m:\u001b[36m116 \u001b[0m | \u001b[33m\u001b[1mFalling back to using the uri: sqlite://.\u001b[0m\n",
-      "\u001b[32m2024-Dec-03 10:16:48.17\u001b[0m| \u001b[1mINFO    \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.base.datalayer\u001b[0m:\u001b[36m68  \u001b[0m | \u001b[1mBuilding Data Layer\u001b[0m\n",
-      "\u001b[32m2024-Dec-03 10:16:48.17\u001b[0m| \u001b[1mINFO    \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.base.build\u001b[0m:\u001b[36m184 \u001b[0m | \u001b[1mConfiguration: \n",
-      " +---------------+-----------+\n",
-      "| Configuration |   Value   |\n",
-      "+---------------+-----------+\n",
-      "|  Data Backend | sqlite:// |\n",
-      "+---------------+-----------+\u001b[0m\n"
+      "\u001b[32m2024-Dec-03 15:27:25.47\u001b[0m| \u001b[1mINFO    \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.misc.plugins\u001b[0m:\u001b[36m13  \u001b[0m | \u001b[1mLoading plugin: mongodb\u001b[0m\n",
+      "\u001b[32m2024-Dec-03 15:27:25.52\u001b[0m| \u001b[1mINFO    \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.base.datalayer\u001b[0m:\u001b[36m68  \u001b[0m | \u001b[1mBuilding Data Layer\u001b[0m\n",
+      "\u001b[32m2024-Dec-03 15:27:25.52\u001b[0m| \u001b[1mINFO    \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.base.build\u001b[0m:\u001b[36m184 \u001b[0m | \u001b[1mConfiguration: \n",
+      " +---------------+-----------------------------------+\n",
+      "| Configuration |               Value               |\n",
+      "+---------------+-----------------------------------+\n",
+      "|  Data Backend | mongodb://localhost:27017/test_db |\n",
+      "+---------------+-----------------------------------+\u001b[0m\n"
      ]
     }
    ],
@@ -98,7 +96,7 @@
     "CFG.output_prefix = OUTPUT_PREFIX\n",
     "CFG.bytes_encoding = 'str'\n",
     "\n",
-    "db = superduper('sqlite://')"
+    "db = superduper()"
    ]
   },
   {
@@ -131,37 +129,6 @@
   {
    "cell_type": "code",
    "execution_count": 4,
-   "id": "9b4dbc0f-5a0a-423c-ad47-f3e4b3cbc950",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from superduper.components.dataset import RemoteData"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "3cef697d-63b7-43d8-af45-b7751323b7f6",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'getter': 'default'}"
-      ]
-     },
-     "execution_count": 5,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "RemoteData._fields"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
    "id": "1ef8dd07-1b47-4dce-84dd-a081d1f5ee9d",
    "metadata": {},
    "outputs": [],
@@ -183,7 +150,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 5,
    "id": "c5965fdf",
    "metadata": {},
    "outputs": [],
@@ -217,7 +184,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 6,
    "id": "2d20eaa0-a416-4483-938e-23f79845739a",
    "metadata": {},
    "outputs": [],
@@ -245,7 +212,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 7,
    "id": "93d21872-d4dc-40dc-abab-fb07ba102ea3",
    "metadata": {},
    "outputs": [],
@@ -264,7 +231,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 8,
    "id": "31900eec-b516-4bef-939e-2e8f46252b12",
    "metadata": {},
    "outputs": [],
@@ -312,7 +279,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 9,
    "id": "a9b1f538-65ca-499e-b6d0-2dd733f81723",
    "metadata": {},
    "outputs": [],
@@ -337,7 +304,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 10,
    "id": "4663fa4b-c2ec-427d-bf8b-b8b109cc2ccf",
    "metadata": {},
    "outputs": [],
@@ -360,7 +327,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 11,
    "id": "509c3505-54c5-4e68-84ec-3df8bea0fd74",
    "metadata": {},
    "outputs": [],
@@ -380,7 +347,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 12,
    "id": "f98e5ff4",
    "metadata": {},
    "outputs": [],
@@ -404,7 +371,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 13,
    "id": "44baeb09-6f35-4cf2-b814-46283a59f7e9",
    "metadata": {},
    "outputs": [],
@@ -431,7 +398,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 14,
    "id": "2d3a0d3a-da1c-41ec-b16c-f281c46ad794",
    "metadata": {},
    "outputs": [],
@@ -450,16 +417,31 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 15,
+   "id": "3202d3de-c2a0-4e0e-82d8-6f2d9e4f4940",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from superduper.components.streamlit import Streamlit\n",
+    "from superduper import Plugin\n",
+    "\n",
+    "from demo import demo_func\n",
+    "\n",
+    "demo = Streamlit('rag', demo_func=demo_func, upstream=[Plugin('demo_dep', path='./demo.py')])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
    "id": "e6787c78-4b14-4a72-818b-450408a74331",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "\u001b[32m2024-Dec-03 10:16:48.96\u001b[0m| \u001b[1mINFO    \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.components.application\u001b[0m:\u001b[36m39  \u001b[0m | \u001b[1mResorting components based on topological order.\u001b[0m\n",
-      "\u001b[32m2024-Dec-03 10:16:48.96\u001b[0m| \u001b[1mINFO    \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.components.application\u001b[0m:\u001b[36m56  \u001b[0m | \u001b[1mNew order of components: ['listener:chunker:f4e285cb704a41c3', 'vector_index:vectorindex:dbd493f4bbe3408a', 'model:simple_rag:a9ff704a42644185']\u001b[0m\n"
+      "\u001b[32m2024-Dec-03 15:27:26.36\u001b[0m| \u001b[1mINFO    \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.components.application\u001b[0m:\u001b[36m39  \u001b[0m | \u001b[1mResorting components based on topological order.\u001b[0m\n",
+      "\u001b[32m2024-Dec-03 15:27:26.36\u001b[0m| \u001b[1mINFO    \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.components.application\u001b[0m:\u001b[36m56  \u001b[0m | \u001b[1mNew order of components: ['listener:chunker:80aa46018b024084', 'vector_index:vectorindex:6c4d69950c9f4c9a', 'model:simple_rag:f836eada8c2b41eb', 'streamlit:rag:7771fea1823b43ef']\u001b[0m\n"
      ]
     }
    ],
@@ -472,13 +454,14 @@
     "        upstream_listener,\n",
     "        vector_index,\n",
     "        rag,\n",
+    "        demo,\n",
     "    ]\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 17,
    "id": "e7c16557-af76-4e70-83d9-2984e19a9554",
    "metadata": {},
    "outputs": [],
@@ -489,12 +472,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 18,
    "id": "2a82ea22-9694-4c65-b72f-c89ae49d1ab2",
    "metadata": {},
    "outputs": [],
    "source": [
     "if APPLY:\n",
+    "    rag = db.load('model', 'simple_rag')\n",
     "    print(rag.predict('Tell me about the project'))"
    ]
   },
@@ -516,20 +500,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 19,
    "id": "2e850c03-33c6-4c88-95d3-d14146a6a0af",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "\u001b[32m2024-Dec-03 10:16:48.97\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.components.listener\u001b[0m:\u001b[36m74  \u001b[0m | \u001b[33m\u001b[1moutput_table not found in listener.dict()\u001b[0m\n",
-      "\u001b[32m2024-Dec-03 10:16:48.97\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.components.listener\u001b[0m:\u001b[36m74  \u001b[0m | \u001b[33m\u001b[1moutput_table not found in listener.dict()\u001b[0m\n",
-      "\u001b[32m2024-Dec-03 10:16:48.97\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.base.document\u001b[0m:\u001b[36m558 \u001b[0m | \u001b[33m\u001b[1mLeaf listener:chunker already exists\u001b[0m\n",
-      "\u001b[32m2024-Dec-03 10:16:48.97\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.components.listener\u001b[0m:\u001b[36m74  \u001b[0m | \u001b[33m\u001b[1moutput_table not found in listener.dict()\u001b[0m\n",
-      "\u001b[32m2024-Dec-03 10:16:48.97\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.base.document\u001b[0m:\u001b[36m558 \u001b[0m | \u001b[33m\u001b[1mLeaf model:chunker already exists\u001b[0m\n",
-      "\u001b[32m2024-Dec-03 10:16:48.98\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.base.document\u001b[0m:\u001b[36m558 \u001b[0m | \u001b[33m\u001b[1mLeaf var-table-name-select-var-id-field-x already exists\u001b[0m\n"
+      "\u001b[32m2024-Dec-03 15:27:26.37\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.components.listener\u001b[0m:\u001b[36m74  \u001b[0m | \u001b[33m\u001b[1moutput_table not found in listener.dict()\u001b[0m\n",
+      "\u001b[32m2024-Dec-03 15:27:26.38\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.components.listener\u001b[0m:\u001b[36m74  \u001b[0m | \u001b[33m\u001b[1moutput_table not found in listener.dict()\u001b[0m\n",
+      "\u001b[32m2024-Dec-03 15:27:26.38\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.base.document\u001b[0m:\u001b[36m558 \u001b[0m | \u001b[33m\u001b[1mLeaf listener:chunker already exists\u001b[0m\n",
+      "\u001b[32m2024-Dec-03 15:27:26.38\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.components.listener\u001b[0m:\u001b[36m74  \u001b[0m | \u001b[33m\u001b[1moutput_table not found in listener.dict()\u001b[0m\n",
+      "\u001b[32m2024-Dec-03 15:27:26.38\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.base.document\u001b[0m:\u001b[36m558 \u001b[0m | \u001b[33m\u001b[1mLeaf model:chunker already exists\u001b[0m\n",
+      "\u001b[32m2024-Dec-03 15:27:26.38\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.base.document\u001b[0m:\u001b[36m558 \u001b[0m | \u001b[33m\u001b[1mLeaf var-table-name-select-var-id-field-x already exists\u001b[0m\n"
      ]
     }
    ],
@@ -542,7 +526,6 @@
     "    template=app,\n",
     "    substitutions={\n",
     "        COLLECTION_NAME: 'table_name',\n",
-    "        'mongodb': 'databackend',\n",
     "        'text-embedding-ada-002': 'embedding_model',\n",
     "        'gpt-3.5-turbo': 'llm_model',\n",
     "    },\n",
@@ -576,47 +559,27 @@
     "        }\n",
     "    },\n",
     "    schema={\n",
-    "        \"id_field\": \"id_field\",\n",
-    "        \"embedding_model\": \"embedding_model\",\n",
-    "        \"llm_model\": \"llm_model\",\n",
-    "        \"table_name\": \"table_name\",\n",
+    "        \"id_field\": \"str\",\n",
+    "        \"embedding_model\": \"str\",\n",
+    "        \"llm_model\": \"str\",\n",
+    "        \"table_name\": \"str\",\n",
+    "        \"databackend\": \"str\",\n",
     "    },\n",
     "    db=db\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
-   "id": "66d77fc5-65e9-4f2d-ac74-13c8fa3c3acc",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Schema(identifier='RemoteData/class_schema', uuid='09196795da834f25', upstream=None, plugins=None, cache=True, status=None, build_variables=None, build_template=None, fields={'getter': DillSerializer(identifier='default', uuid='0e8771fabb3a4624', upstream=None, plugins=None, cache=True, status=None, build_variables=None, build_template=None)})"
-      ]
-     },
-     "execution_count": 21,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "template.default_table.data.build_class_schema()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 20,
    "id": "8924ba0d-7c01-4d6c-87fb-245531db7506",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "\u001b[32m2024-Dec-03 10:16:48.99\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.base.document\u001b[0m:\u001b[36m558 \u001b[0m | \u001b[33m\u001b[1mLeaf str already exists\u001b[0m\n"
+      "\u001b[32m2024-Dec-03 15:27:26.39\u001b[0m| \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mDuncans-MBP.fritz.box\u001b[0m| \u001b[36msuperduper.base.document\u001b[0m:\u001b[36m558 \u001b[0m | \u001b[33m\u001b[1mLeaf str already exists\u001b[0m\n"
      ]
     }
    ],