Provide paginate_messages option to allow use of previous behavior

Author: mkupferman

docs/source/usage.md

@@ -292,6 +292,8 @@ The full set of configuration options are:
       (Default: `https://www.googleapis.com/auth/gmail.modify`)
   - `oauth2_port` - int: The TCP port for the local server to
       listen on for the OAuth2 response (Default: `8080`)
+  - `paginate_messages` - bool: When `True`, fetch all applicable Gmail messages.
+      When `False`, only fetch up to 100 new messages per run (Default: `True`)
 - `log_analytics`
   - `client_id` - str: The app registration's client ID
   - `client_secret` - str: The app registration's client secret

parsedmarc/cli.py

@@ -335,6 +335,7 @@ def process_reports(reports_):
                      gmail_api_credentials_file=None,
                      gmail_api_token_file=None,
                      gmail_api_include_spam_trash=False,
+                     gmail_api_paginate_messages=True,
                      gmail_api_scopes=[],
                      gmail_api_oauth2_port=8080,
                      log_file=args.log_file,
@@ -750,6 +751,8 @@ def process_reports(reports_):
                 gmail_api_config.get("token_file", ".token")
             opts.gmail_api_include_spam_trash = \
                 gmail_api_config.getboolean("include_spam_trash", False)
+            opts.gmail_api_paginate_messages = \
+                gmail_api_config.getboolean("paginate_messages", True)
             opts.gmail_api_scopes = \
                 gmail_api_config.get("scopes",
                                      default_gmail_api_scope)
@@ -1009,6 +1012,7 @@ def process_reports(reports_):
                 token_file=opts.gmail_api_token_file,
                 scopes=opts.gmail_api_scopes,
                 include_spam_trash=opts.gmail_api_include_spam_trash,
+                paginate_messages=opts.gmail_api_paginate_messages,
                 reports_folder=opts.mailbox_reports_folder,
                 oauth2_port=opts.gmail_api_oauth2_port
             )

parsedmarc/mail/gmail.py

@@ -42,11 +42,13 @@ def __init__(self,
                  scopes: List[str],
                  include_spam_trash: bool,
                  reports_folder: str,
-                 oauth2_port: int):
+                 oauth2_port: int,
+                 paginate_messages: bool):
         creds = _get_creds(token_file, credentials_file, scopes, oauth2_port)
         self.service = build('gmail', 'v1', credentials=creds)
         self.include_spam_trash = include_spam_trash
         self.reports_label_id = self._find_label_id_for_label(reports_folder)
+        self.paginate_messages = paginate_messages
 
     def create_folder(self, folder_name: str):
         # Gmail doesn't support the name Archive
@@ -81,7 +83,7 @@ def _fetch_all_message_ids(self, reports_label_id, page_token=None):
         for message in messages:
             yield message["id"]
 
-        if "nextPageToken" in results:
+        if "nextPageToken" in results and self.paginate_messages:
             yield from self._fetch_all_message_ids(
                 reports_label_id, results["nextPageToken"]
             )