Exit the process if status is finished and not in the middle of completing (#2378)

This solves an issue where a run is system failed from the platform and 
the worker is never shut down in the cluster.

Author: matt-aitken

packages/cli-v3/src/entryPoints/managed/execution.ts

@@ -76,6 +76,8 @@ export class RunExecution {
   private isShuttingDown = false;
   private shutdownReason?: string;
 
+  private isCompletingRun = false;
+
   private supervisorSocket: SupervisorSocket;
   private notifier?: RunNotifier;
   private metadataClient?: MetadataClient;
@@ -292,7 +294,13 @@ export class RunExecution {
       case "FINISHED": {
         this.sendDebugLog("run is finished", snapshotMetadata);
 
-        // This can sometimes be called before the handleCompletionResult, so we don't need to do anything here
+        // We are finishing the run in handleCompletionResult, so we don't need to do anything here
+        if (this.isCompletingRun) {
+          this.sendDebugLog("run is finished but we're completing it, skipping", snapshotMetadata);
+          return;
+        }
+
+        await this.exitTaskRunProcessWithoutFailingRun({ flush: true, reason: "re-queued" });
         return;
       }
       case "QUEUED_EXECUTING":
@@ -377,6 +385,9 @@ export class RunExecution {
       throw new Error("Cannot start attempt: missing run or snapshot manager");
     }
 
+    // Reset this for the new attempt
+    this.isCompletingRun = false;
+
     this.sendDebugLog("starting attempt", { isWarmStart: String(isWarmStart) });
 
     const attemptStartedAt = Date.now();
@@ -655,6 +666,8 @@ export class RunExecution {
       throw new Error("cannot complete run: missing run or snapshot manager");
     }
 
+    this.isCompletingRun = true;
+
     const completionResult = await this.httpClient.completeRunAttempt(
       this.runFriendlyId,
       this.snapshotManager.snapshotId,