Catch errors in stream

The iter_lines() call in Twarc2.stream needs to catch more than HTTP errors. See #471 for details.
DocNow · Jul 1, 2021 · cea8138 · cea8138
1 parent 2e6fa20
commit cea8138
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 4 deletions.
diff --git a/twarc/client2.py b/twarc/client2.py
@@ -503,13 +503,18 @@ def sample(self, event=None, record_keepalive=False):
                                     time.sleep(5)
                                     break
 
-            except requests.exceptions.HTTPError as e:
+            except requests.exceptions.RequestException as e:
                 errors += 1
-                log.error("caught http error %s on %s try", e, errors)
+                log.error("caught request error %s on %s try", e, errors)
+
                 if self.http_errors and errors == self.http_errors:
                     log.warning("too many errors")
                     raise e
-                if e.response.status_code == 420:
+
+                if (
+                    isinstance(e, requests.exceptions.HTTPError)
+                    and response.status_code == 420
+                ):
                     if interruptible_sleep(errors * 60, event):
                         log.info("stopping filter")
                         return

diff --git a/twarc/version.py b/twarc/version.py
@@ -1 +1 @@
-version = "2.3.2"
+version = "2.3.3"