fix(run-engine): waitpoint update misleading error logs

The `completeWaitpoint` method ignores Prisma update errors of type `P2025` to
handle concurrent attempts to complete a waitpoint token. We have an
error hook on the Prisma client which logs these errors though.

This PR switches to `updateMany` which doesn't throw an error if the
waitpoint is in an unexpected state. The behavior remains otherwise
unchanged. We currently continue even if the waitpoint is not in `PENDING`,
which could lead to race conditions in certain corner cases, though very
unlikely. We added a log line to see how often concurrent completion
attempts happen.

Author: myftija

internal-packages/run-engine/src/engine/systems/waitpointSystem.ts

@@ -79,8 +79,8 @@ export class WaitpointSystem {
     };
   }): Promise<Waitpoint> {
     // 1. Complete the Waitpoint (if not completed)
-    let [waitpointError, waitpoint] = await tryCatch(
-      this.$.prisma.waitpoint.update({
+    const [updateError, updateResult] = await tryCatch(
+      this.$.prisma.waitpoint.updateMany({
         where: { id, status: "PENDING" },
         data: {
           status: "COMPLETED",
@@ -92,29 +92,32 @@ export class WaitpointSystem {
       })
     );
 
-    if (waitpointError) {
-      if (
-        waitpointError instanceof Prisma.PrismaClientKnownRequestError &&
-        waitpointError.code === "P2025"
-      ) {
-        waitpoint = await this.$.prisma.waitpoint.findFirst({
-          where: { id },
-        });
-      } else {
-        this.$.logger.log("completeWaitpoint: error updating waitpoint:", { waitpointError });
-        throw waitpointError;
-      }
+    if (updateError) {
+      this.$.logger.error("completeWaitpoint: error updating waitpoint:", { updateError });
+      throw updateError;
     }
 
+    if (updateResult.count === 0) {
+      this.$.logger.info(
+        "completeWaitpoint: attempted to complete a waitpoint that is not PENDING",
+        { waitpointId: id }
+      );
+    }
+
+    const waitpoint = await this.$.prisma.waitpoint.findFirst({
+      where: { id },
+    });
+
     if (!waitpoint) {
-      throw new Error(`Waitpoint ${id} not found`);
+      this.$.logger.error("completeWaitpoint: waitpoint not found", { waitpointId: id });
+      throw new Error("Waitpoint not found");
     }
 
     if (waitpoint.status !== "COMPLETED") {
       this.$.logger.error(`completeWaitpoint: waitpoint is not completed`, {
         waitpointId: id,
       });
-      throw new Error(`Waitpoint ${id} is not completed`);
+      throw new Error("Waitpoint not completed");
     }
 
     // 2. Find the TaskRuns blocked by this waitpoint