Fix several restore and resume bugs (#1418)

* try to correct resume messages with missing checkpoint

* prevent creating checkpoints for outdated task waits

* prevent creating checkpoints for outdated batch waits

* use heartbeats to check for and clean up any leftover containers

* lint

* improve exec logging

* improve resume attempt logs

* fix for resuming parents of canceled child runs

* separate SIGTERM from maybe OOM errors

* pretty errors can have magic dashboard links

* prevent uncancellable checkpoints

* simplify task run error code enum export

* grab the last, not the first child run

* Revert "prevent creating checkpoints for outdated batch waits"

This reverts commit f2b5c2a.

* Revert "grab the last, not the first child run"

This reverts commit 89ec5c8.

* Revert "prevent creating checkpoints for outdated task waits"

This reverts commit 11066b4.

* more logs for resume message handling

* add magic error link comment

* add changeset

Author: nicktrn

.changeset/many-plants-destroy.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+SIGTERM detection and prettier errors

apps/coordinator/src/checkpointer.ts

@@ -436,7 +436,10 @@ export class Checkpointer {
         this.#logger.error("Error during cleanup", { ...metadata, error });
       }
 
-      this.#abortControllers.delete(runId);
+      // Ensure only the current controller is removed
+      if (this.#abortControllers.get(runId) === controller) {
+        this.#abortControllers.delete(runId);
+      }
       controller.signal.removeEventListener("abort", onAbort);
     };
 

apps/coordinator/src/exec.ts

@@ -64,7 +64,18 @@ export class Exec {
       command,
       argsRaw: args,
       argsTrimmed,
-      ...output,
+      globalOpts: {
+        trimArgs: this.trimArgs,
+        neverThrow: this.neverThrow,
+        hasAbortSignal: !!this.abortSignal,
+      },
+      localOpts: opts,
+      stdout: output.stdout,
+      stderr: output.stderr,
+      pid: result.pid,
+      exitCode: result.exitCode,
+      aborted: result.aborted,
+      killed: result.killed,
     };
 
     if (this.logOutput) {

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx

@@ -1,4 +1,10 @@
-import { CheckIcon, ClockIcon, CloudArrowDownIcon, QueueListIcon } from "@heroicons/react/20/solid";
+import {
+  CheckIcon,
+  ClockIcon,
+  CloudArrowDownIcon,
+  EnvelopeIcon,
+  QueueListIcon,
+} from "@heroicons/react/20/solid";
 import { Link } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import {
@@ -13,6 +19,7 @@ import { typedjson, useTypedFetcher } from "remix-typedjson";
 import { ExitIcon } from "~/assets/icons/ExitIcon";
 import { CodeBlock } from "~/components/code/CodeBlock";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
+import { Feedback } from "~/components/Feedback";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { Callout } from "~/components/primitives/Callout";
 import { DateTime, DateTimeAccurate } from "~/components/primitives/DateTime";
@@ -963,11 +970,26 @@ function RunError({ error }: { error: TaskRunError }) {
         <div className="flex flex-col gap-2 rounded-sm border border-rose-500/50 px-3 pb-3 pt-2">
           <Header3 className="text-rose-500">{name}</Header3>
           {enhancedError.message && <Callout variant="error">{enhancedError.message}</Callout>}
-          {enhancedError.link && (
-            <Callout variant="docs" to={enhancedError.link.href}>
-              {enhancedError.link.name}
-            </Callout>
-          )}
+          {enhancedError.link &&
+            (enhancedError.link.magic === "CONTACT_FORM" ? (
+              <Feedback
+                button={
+                  <Button
+                    variant="tertiary/medium"
+                    LeadingIcon={EnvelopeIcon}
+                    leadingIconClassName="text-blue-400"
+                    fullWidth
+                    textAlignLeft
+                  >
+                    {enhancedError.link.name}
+                  </Button>
+                }
+              />
+            ) : (
+              <Callout variant="docs" to={enhancedError.link.href}>
+                {enhancedError.link.name}
+              </Callout>
+            ))}
           {enhancedError.stackTrace && (
             <CodeBlock
               showCopyButton={false}

apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts

@@ -136,7 +136,7 @@ export class EnvironmentVariablesRepository implements Repository {
 
     try {
       for (const variable of values) {
-      const result = await $transaction(this.prismaClient, async (tx) => {
+        const result = await $transaction(this.prismaClient, async (tx) => {
           const environmentVariable = await tx.environmentVariable.upsert({
             where: {
               projectId_key: {

apps/webapp/app/v3/handleSocketIo.server.ts

@@ -195,7 +195,10 @@ function createCoordinatorNamespace(io: Server) {
           const service = new CreateTaskRunAttemptService();
           const { attempt } = await service.call(message.runId, environment, false);
 
-          const payload = await sharedQueueTasks.getExecutionPayloadFromAttempt(attempt.id, true);
+          const payload = await sharedQueueTasks.getExecutionPayloadFromAttempt({
+            id: attempt.id,
+            setToExecuting: true,
+          });
 
           if (!payload) {
             logger.error("Failed to retrieve payload after attempt creation", message);