fix(engine): carry over completed waitpoints on PENDING_EXECUTING stalls (#2503)

Author: ericallam

internal-packages/run-engine/src/engine/index.ts

@@ -1238,6 +1238,8 @@ export class RunEngine {
             },
             orgId: latestSnapshot.organizationId,
             projectId: latestSnapshot.projectId,
+            checkpointId: latestSnapshot.checkpointId ?? undefined,
+            completedWaitpoints: latestSnapshot.completedWaitpoints,
             error: {
               type: "INTERNAL_ERROR",
               code: "TASK_RUN_DEQUEUED_MAX_RETRIES",

internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts

@@ -1139,6 +1139,7 @@ export class RunAttemptSystem {
     workerId,
     runnerId,
     checkpointId,
+    completedWaitpoints,
     tx,
   }: {
     run: TaskRun;
@@ -1154,6 +1155,10 @@ export class RunAttemptSystem {
     runnerId?: string;
     checkpointId?: string;
     tx?: PrismaClientOrTransaction;
+    completedWaitpoints?: {
+      id: string;
+      index?: number;
+    }[];
   }): Promise<{ wasRequeued: boolean } & ExecutionResult> {
     const prisma = tx ?? this.$.prisma;
 
@@ -1201,6 +1206,7 @@ export class RunAttemptSystem {
         workerId,
         runnerId,
         checkpointId,
+        completedWaitpoints,
       });
 
       return {