Fixes #38

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## v24.19.0
+
+- Removed a stray `break` in S3 file listing that was preventing fetching more than the first 1000 records
+
 ## v24.16.0
 
 - Added a wider window for refreshing token. If it's within 60 seconds of expiry when checking we will refresh it, so handle instances where there's a delay renewing.

src/opentaskpy/addons/aws/remotehandlers/s3.py

@@ -272,7 +272,8 @@ def list_files(
                         kwargs["ContinuationToken"] = response["NextContinuationToken"]
                     except KeyError:
                         break
-                break
+                else:
+                    break
         except Exception as e:  # pylint: disable=broad-exception-caught
             self.logger.error(f"Error listing files: {self.spec['bucket']}")
             self.logger.exception(e)

tests/test_remotehandler_s3_transfer.py

@@ -46,6 +46,21 @@
     },
 }
 
+s3_file_watch_pagination_task_definition = {
+    "type": "transfer",
+    "source": {
+        "bucket": BUCKET_NAME,
+        "directory": "src",
+        "fileRegex": ".*-xxxx\\.txt",
+        "protocol": {
+            "name": "opentaskpy.addons.aws.remotehandlers.s3.S3Transfer",
+        },
+        "fileWatch": {
+            "timeout": 10,
+        },
+    },
+}
+
 
 s3_age_conditions_task_definition = {
     "type": "transfer",
@@ -991,6 +1006,31 @@ def test_s3_file_watch_custom_creds(
     assert transfer_obj.run()
 
 
+def test_s3_file_watch_pagination(s3_client, setup_bucket, tmp_path):
+    transfer_obj = transfer.Transfer(
+        None, "s3-file-watch-pagination", s3_file_watch_pagination_task_definition
+    )
+
+    # Create a file to watch for with the current date
+    datestamp = datetime.datetime.now().strftime("%Y%m%d")
+
+    # Write 1010 files locally
+    for i in range(1010):
+        fs.create_files([{f"{tmp_path}/{datestamp}-{i}.txt": {"content": "test1234"}}])
+        create_s3_file(
+            s3_client, f"{tmp_path}/{datestamp}-{i}.txt", f"src/{datestamp}-{i}.txt"
+        )
+
+    # Now write another
+    fs.create_files([{f"{tmp_path}/{datestamp}-xxxx.txt": {"content": "test1234"}}])
+    create_s3_file(
+        s3_client, f"{tmp_path}/{datestamp}-xxxx.txt", f"src/{datestamp}-xxxx.txt"
+    )
+
+    # File should be found
+    assert transfer_obj.run()
+
+
 def create_s3_file(s3_client, local_file, object_key):
     s3_client.put_object(
         Bucket=BUCKET_NAME,