Crawler bug fixing

polterguy · Oct 11, 2023 · 1129389 · 1129389
1 parent 4d278fb
commit 1129389
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 30 deletions.
diff --git a/backend/files/system/openai/magic.startup/crawling/magic.ai.crawl-site.hl b/backend/files/system/openai/magic.startup/crawling/magic.ai.crawl-site.hl
@@ -295,35 +295,11 @@ slots.create:magic.ai.crawl-site
                    */
                   for-each:x:@signal/*
 
-                     // Verifying we have not already scraped URL.
-                     .exists:bool:false
-                     for-each:x:@.done/*
-
-                        // Checking if URL exists in [.done] collection.
-                        if
-                           eq:x:@.dp/#
-                              get-value:x:@for-each/@.dp/#
-                           .lambda
-
-                              // URL has already been imported.
-                              set-value:x:@.exists
-                                 .:bool:true
-                     for-each:x:@.urls/*
-
-                        // Checking if URL exists in [.done] collection.
-                        if
-                           eq:x:@.dp/#
-                              get-value:x:@for-each/@.dp/#
-                           .lambda
-
-                              // URL has already been imported.
-                              set-value:x:@.exists
-                                 .:bool:true
-
-                     // Checking if URL has been imported before.
+                     // Checking if URL has been imported or added before.
                      if
-                        eq:x:@.exists
-                           .:bool:false
+                        and
+                           not-exists:x:@.done/*/={@.dp/#}
+                           not-exists:x:@.urls/*/={@.dp/#}
                         .lambda
 
                            // Adding URL to [.urls] collection.

diff --git a/backend/files/system/openai/magic.startup/crawling/magic.ai.url.get.hl b/backend/files/system/openai/magic.startup/crawling/magic.ai.url.get.hl
@@ -48,7 +48,7 @@ slots.create:magic.ai.url.get
             url:x:@.arguments/*/url
             status:x:@http.get
          strings.concat
-            .:"Something went wrong while trying to get URL, HTTP invocation returned status code "
+            .:"HTTP invocation returned status code "
             get-value:x:@http.get
          throw:x:-
 

diff --git a/backend/files/system/openai/magic.startup/crawling/magic.ai.url.scrape.hl b/backend/files/system/openai/magic.startup/crawling/magic.ai.url.scrape.hl
@@ -309,6 +309,28 @@ slots.create:magic.ai.url.scrape
 
                                  // Checking that above invocation returned success.
                                  if
+                                    eq:x:@http.post
+                                       .:int:400
+                                    .lambda
+
+                                       // Probably too many tokens to summarize.
+                                       sockets.signal:x:@.arguments/*/feedback-channel
+                                          roles:root
+                                          args
+                                             message:Could not summarize snippet, check your log for details
+                                             type:warning
+                                       sleep:100
+                                       lambda2hyper:x:@http.post
+                                       log.error:Something went wrong while invoking OpenAI
+                                          message:x:@http.post/*/content/*/error/*/message
+                                          status:x:@http.post
+                                          error:x:@lambda2hyper
+
+                                       // Making sure we don't try again.
+                                       set-value:x:@.no
+                                          .:int:0
+
+                                 else-if
                                     and
                                        mte:x:@http.post
                                           .:int:200
@@ -680,7 +702,7 @@ slots.create:magic.ai.url.scrape
          get-value:x:@.arguments/*/message
          .:"'"
       unwrap:x:+/**
-      sockets.signal:x:@.arguments/*/feedback-channel
+      sockets.signal:x:@.arguments/@.arguments/*/feedback-channel
          roles:root
          args
             message:x:@strings.concat