From 485bb670baa2425f8b6d7da7698dc677666f09da Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 16 Jul 2025 16:46:59 +0200 Subject: [PATCH 1/3] Add check for resumeTokens going back in time. --- modules/module-mongodb/src/replication/ChangeStream.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index f85de233b..9d0b19821 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -947,6 +947,16 @@ export class ChangeStream { timestamp: changeDocument.clusterTime!, resume_token: changeDocument._id }); + if (batch.lastCheckpointLsn != null && lsn < batch.lastCheckpointLsn) { + // Checkpoint out of order - should never happen with MongoDB. + // If it does happen, we throw an error to stop the replication - restarting should recover. + // Since we use batch.lastCheckpointLsn for the next resumeAfter, this should not result in an infinite loop. + // This is a workaround for the issue below, but we can keep this as a safety-check even if the issue is fixed. + // Driver issue report: https://jira.mongodb.org/browse/NODE-7042 + throw new ReplicationAssertionError( + `Change resumeToken ${(changeDocument._id as any)._data} (${timestampToDate(changeDocument.clusterTime!).toISOString()}) is less than last checkpoint LSN ${batch.lastCheckpointLsn}. Restarting replication.` + ); + } if (waitForCheckpointLsn != null && lsn >= waitForCheckpointLsn) { waitForCheckpointLsn = null; From d057e37faf7c41f393a8297712cb24b2f66551cc Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 16 Jul 2025 16:49:05 +0200 Subject: [PATCH 2/3] Reduce retry delay on general MongoDB change stream errors. --- modules/module-mongodb/src/replication/MongoErrorRateLimiter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-mongodb/src/replication/MongoErrorRateLimiter.ts b/modules/module-mongodb/src/replication/MongoErrorRateLimiter.ts index 17b65c66c..3c770fb9b 100644 --- a/modules/module-mongodb/src/replication/MongoErrorRateLimiter.ts +++ b/modules/module-mongodb/src/replication/MongoErrorRateLimiter.ts @@ -28,7 +28,7 @@ export class MongoErrorRateLimiter implements ErrorRateLimiter { // Could be fail2ban or similar this.setDelay(120_000); } else { - this.setDelay(30_000); + this.setDelay(5_000); } } From 5c3e6edbe6e249b26249616920d8595788d09cd0 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 16 Jul 2025 16:49:57 +0200 Subject: [PATCH 3/3] Add changeset. --- .changeset/sweet-years-tickle.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changeset/sweet-years-tickle.md diff --git a/.changeset/sweet-years-tickle.md b/.changeset/sweet-years-tickle.md new file mode 100644 index 000000000..02cc3f6ef --- /dev/null +++ b/.changeset/sweet-years-tickle.md @@ -0,0 +1,8 @@ +--- +'@powersync/service-module-mongodb': patch +'@powersync/lib-service-mongodb': patch +'@powersync/service-core': patch +'@powersync/service-image': patch +--- + +[MongoDB Replication] Fix resumeTokens going back in time on busy change streams.