BETA release of "query" endpoint

backend/files/system/magic.startup/magic.http.duckduckgo-and-scrape.hl

@@ -19,6 +19,21 @@ slots.create:magic.http.duckduckgo-and-scrape
       get-nodes:x:@.arguments/*
    signal:magic.http.duckduckgo-search
 
+   // Checking if we've got a session, and if so, notifying user using web sockets.
+   if
+      exists:x:@.arguments/*/session
+      .lambda
+         for-each:x:@signal/*/result/*
+            strings.concat
+               .:"Scraping: "
+               get-value:x:@.dp/#/*/url
+            unwrap:x:+/**
+            sockets.signal:x:@.arguments/*/session
+               args
+                  message:x:@strings.concat
+                  type:system
+            sleep:100
+
    // Building our execution object that fetches all URLs simultaneously in parallel.
    .exe
 
@@ -36,12 +51,25 @@ slots.create:magic.http.duckduckgo-and-scrape
                signal:magic.http.scrape-url
                   url:x:@.reference/*/url
                   semantics:bool:true
+               if
+                  exists:x:@.reference/*/session
+                  .lambda
+                     strings.concat
+                        .:"Done scraping "
+                        get-value:x:@.reference/*/url
+                     unwrap:x:+/**
+                     sockets.signal:x:@.reference/*/session
+                        args
+                           message:x:@strings.concat
+                           type:system
             .catch
                log.error:Could not scrape URL
                   url:x:@.reference/*/url
                   message:x:@.arguments/*/message
 
       // Adding URL and title as reference to currently iterated [fork].
+      add:x:+/+/*
+         get-nodes:x:@.arguments/*/session
       unwrap:x:+/*/*
       add:x:@.cur/*/fork/*/.reference
          .
@@ -54,6 +82,17 @@ slots.create:magic.http.duckduckgo-and-scrape
 
    // Executing [.exe] retrieving all URLs in parallel.
    eval:x:@.exe
+   remove-nodes:x:@.exe/**/session
+
+   // Signaling caller on sockets.
+   if
+      exists:x:@.arguments/*/session
+      .lambda
+         sockets.signal:x:@.arguments/*/session
+            args
+               message:Building context
+               type:system
+         sleep:100
 
    /*
     * Iterating through each above result,

backend/files/system/magic.startup/magic.http.duckduckgo-search.hl

@@ -21,6 +21,19 @@ slots.create:magic.http.duckduckgo-search
          .:"https://html.duckduckgo.com/html/?q="
          strings.url-encode:x:@.arguments/*/query
 
+   // Checking if caller provided a session argument, and if so, signaling client using web sockets.
+   if
+      exists:x:@.arguments/*/session
+      .lambda
+         strings.concat
+            .:"Invoking DuckDuckGo at: "
+            get-value:x:@.url
+         unwrap:x:+/**
+         sockets.signal:x:@.arguments/*/session
+            args
+               message:x:@strings.concat
+               type:system
+
    // Invoking slot responsible for retrieving search results.
    unwrap:x:+/*
    signal:magic.http.get-with-cache

backend/files/system/openai/magic.startup/magic.ai.create-search-context.hl

@@ -12,9 +12,11 @@ slots.create:magic.ai.create-search-context
    validators.mandatory:x:@.arguments/*/max_tokens
    validators.integer:x:@.arguments/*/max_tokens
       min:250
-      max:8000
+      max:15000
 
    // Invoking slot responsible for searching DuckDuckGo and scraping results.
+   add:x:./*/signal
+      get-nodes:x:@.arguments/*/session
    unwrap:x:+/*/query
    signal:magic.http.duckduckgo-and-scrape
       query:x:@.arguments/*/query

backend/files/system/openai/query.get.hl

@@ -18,16 +18,25 @@
 
 // Sanity checking invocation.
 validators.mandatory:x:@.arguments/*/session
+validators.string:x:@.arguments/*/session
+   min:5
+   max:50
 validators.mandatory:x:@.arguments/*/query
+validators.string:x:@.arguments/*/query
+   min:2
+   max:200
 validators.mandatory:x:@.arguments/*/instruction
+validators.string:x:@.arguments/*/instruction
+   min:5
+   max:750
 
 /*
  * Checking if backend is configured to use reCAPTCHA, and if so,
  * making sure we ensure the frontend supplied a [recaptcha_response],
  * and invoking slot responsible for verifying reCAPTCHA value.
  */
 config.get:"magic:auth:recaptcha:key"
-.if
+if
    and
       neq:x:@config.get
          .
@@ -59,15 +68,15 @@ if
 
       // Caller supplied a URL.
       strings.concat
-         .:"Scraping URL: '"
+         .:"Scraping URL: "
          get-value:x:@.arguments/*/query
-         .:"'"
       unwrap:x:+/**
       sockets.signal:x:@.arguments/*/session
          args
             message:x:@strings.concat
             type:system
-
+      sleep:100
+
       // Scraping specified URL.
       unwrap:x:+/*
       signal:magic.ai.url.get
@@ -82,14 +91,14 @@ if
          semantic:bool:true
          html:x:@signal/@signal
 
-      // Building our context making sure we don't overflow 4,000 tokens.
+      // Building our context making sure we don't overflow 10,000 tokens.
       .tmp:
       while
          and
             exists:x:@signal/*/snippets/0
             lt
                openai.tokenize:x:@.tmp
-               .:int:4000
+               .:int:5000
          .lambda
 
             // Updating [.context] to current value of [.tmp].
@@ -144,12 +153,14 @@ else
       args
          message:x:@strings.concat
          type:system
+   sleep:100
 
    // Caller supplied a search query.
    unwrap:x:+/*
    signal:magic.ai.create-search-context
       query:x:@.arguments/*/query
-      max_tokens:int:4000
+      max_tokens:int:5000
+      session:x:@.arguments/*/session
    set-value:x:@.context
       strings.trim:x:@signal
 
@@ -161,6 +172,22 @@ set-value:x:@.context
 insert-before:x:./*/fork/0
    get-nodes:x:@.arguments
    get-nodes:x:@.context
+
+// Signaling caller on sockets.
+openai.tokenize:x:@.arguments/*/instruction
+openai.tokenize:x:@.context
+strings.concat
+   .:"Invoking OpenAI with "
+   get-value:x:@openai.tokenize
+   .:" context tokens and "
+   get-value:x:@openai.tokenize/@openai.tokenize
+   .:" instruction tokens"
+unwrap:x:+/**
+sockets.signal:x:@.arguments/*/session
+   args
+      message:x:@strings.concat
+      type:system
+sleep:100
 
 // Creating a fork to take advantage of SSE features of OpenAI.
 fork
@@ -260,8 +287,8 @@ fork
             Content-Type:application/json
             Accept:text/event-stream
          payload
-            model:gpt-4
-            max_tokens:int:4100
+            model:gpt-3.5-turbo-16k
+            max_tokens:int:4000
             temperature:decimal:0.3
             stream:bool:true
             messages