Clean up throttle rates, throttle the agent

DeabLabs · Oct 19, 2023 · 01cbad4 · 01cbad4
1 parent d661c52
commit 01cbad4
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 2 deletions.
diff --git a/apps/messages/package.json b/apps/messages/package.json
@@ -13,6 +13,7 @@
     "@neondatabase/serverless": "^0.6.0",
     "database": "*",
     "dotenv": "^16.3.1",
+    "just-throttle": "^4.2.0",
     "mclient": "*",
     "openai": "^4.12.3",
     "partyrpc": "*",

diff --git a/apps/messages/src/agent.ts b/apps/messages/src/agent.ts
@@ -11,6 +11,7 @@ import { Pool } from '@neondatabase/serverless';
 import { getServerlessDb } from 'database/drizzle';
 import invariant from 'tiny-invariant';
 import type { ChatCompletionMessageParam } from 'openai/resources';
+import throttle from 'just-throttle';
 
 const connectionRequestBodySchema = valibot.object({
   /** The connect action */
@@ -190,18 +191,34 @@ export default class Agent implements Party.Server {
         stream: true
       });
 
+      const sendEdit = throttle(
+        (content: string) => {
+          this.client?.send({
+            type: 'editMessage',
+            messageId: newMessagePlaceholder.id,
+            content,
+            updatedAt: Dates.getCurrentDateInUTC()
+          });
+        },
+        128,
+        { trailing: true, leading: true }
+      );
+
       let text = '';
       for await (const part of stream) {
         const delta = part.choices?.[0]?.delta?.content;
-        if (delta) {
+        if (delta !== null && delta !== undefined) {
           text = `${text}${delta}`;
+          console.log(delta);
+          sendEdit(text);
+        } else {
+          // dump the final text when done
           this.client?.send({
             type: 'editMessage',
             messageId: newMessagePlaceholder.id,
             content: text,
             updatedAt: Dates.getCurrentDateInUTC()
           });
-        } else {
           this.responding = false;
         }
       }