fix(runner): prevent retry immediately race condition which can cause stuck runs that end up being system failures (#2402)

Author: ericallam

.changeset/loud-rules-dream.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+fix(runner): prevent retry immediately race condition which can cause stuck runs that end up being system failures

packages/cli-v3/src/entryPoints/managed/execution.ts

@@ -77,6 +77,7 @@ export class RunExecution {
   private shutdownReason?: string;
 
   private isCompletingRun = false;
+  private ignoreSnapshotChanges = false;
 
   private supervisorSocket: SupervisorSocket;
   private notifier?: RunNotifier;
@@ -237,6 +238,16 @@ export class RunExecution {
       completedWaitpoints: completedWaitpoints.length,
     };
 
+    if (this.ignoreSnapshotChanges) {
+      this.sendDebugLog("processSnapshotChange: ignoring snapshot change", {
+        incomingSnapshotId: snapshot.friendlyId,
+        completedWaitpoints: completedWaitpoints.length,
+        currentAttemptNumber: this.currentAttemptNumber,
+        newAttemptNumber: run.attemptNumber,
+      });
+      return;
+    }
+
     if (!this.snapshotManager) {
       this.sendDebugLog("handleSnapshotChange: missing snapshot manager", snapshotMetadata);
       return;
@@ -808,7 +819,9 @@ export class RunExecution {
     }
 
     // Start and execute next attempt
-    const [startError, start] = await tryCatch(this.startAttempt({ isWarmStart: true }));
+    const [startError, start] = await tryCatch(
+      this.enableIgnoreSnapshotChanges(() => this.startAttempt({ isWarmStart: true }))
+    );
 
     if (startError) {
       this.sendDebugLog("failed to start attempt for retry", { error: startError.message });
@@ -829,6 +842,15 @@ export class RunExecution {
     }
   }
 
+  private async enableIgnoreSnapshotChanges<T>(fn: () => Promise<T>): Promise<T> {
+    this.ignoreSnapshotChanges = true;
+    try {
+      return await fn();
+    } finally {
+      this.ignoreSnapshotChanges = false;
+    }
+  }
+
   /**
    * Restores a suspended execution from PENDING_EXECUTING
    */

references/hello-world/src/trigger/attemptFailures.ts

@@ -0,0 +1,50 @@
+import { task } from "@trigger.dev/sdk";
+import { setTimeout } from "timers/promises";
+
+export const attemptFailures = task({
+  id: "attempt-failures",
+  retry: {
+    maxAttempts: 3,
+    minTimeoutInMs: 500,
+    maxTimeoutInMs: 1000,
+    factor: 1.5,
+  },
+  run: async (payload: any, { ctx }) => {
+    await setTimeout(5);
+
+    await attemptFailureSubtask.triggerAndWait({}).unwrap();
+  },
+});
+
+export const attemptFailureSubtask = task({
+  id: "attempt-failure-subtask",
+  retry: {
+    maxAttempts: 1,
+  },
+  run: async (payload: any, { ctx }) => {
+    await setTimeout(20_000);
+
+    throw new Error("Forced error to cause a retry");
+  },
+});
+
+export const attemptFailures2 = task({
+  id: "attempt-failures-2",
+  retry: {
+    maxAttempts: 3,
+    minTimeoutInMs: 500,
+    maxTimeoutInMs: 1000,
+    factor: 1.5,
+  },
+  run: async (payload: any, { ctx }) => {
+    if (ctx.attempt.number <= 2) {
+      throw new Error("Forced error to cause a retry");
+    }
+
+    await setTimeout(10_000);
+
+    return {
+      success: true,
+    };
+  },
+});