[server] Tone down non-error log messages during workspace start/reconcile (#18720)

Author: geropl

components/server/src/jobs/runner.ts

@@ -108,7 +108,7 @@ export class JobRunner {
                 });
             });
         } catch (err) {
-            if (RedisMutex.isLockError(err)) {
+            if (RedisMutex.isLockedError(err)) {
                 log.debug(
                     `Failed to acquire lock for job ${job.name}. Likely another instance already holds the lock.`,
                     err,

components/server/src/redis/mutex.ts

@@ -45,7 +45,12 @@ export class RedisMutex {
         return this.client().using(resources, duration, settings, routine);
     }
 
-    public static isLockError(err: any): boolean {
-        return err instanceof ResourceLockedError || err instanceof ExecutionError;
+    public static isLockedError(err: any): boolean {
+        return (
+            err instanceof ResourceLockedError ||
+            // Should be a ResourceLockedError as well, but is ExecutionError in v5.x: https://github.com/mike-marcacci/node-redlock/issues/168
+            (err instanceof ExecutionError &&
+                err.message.includes("unable to achieve a quorum during its retry window"))
+        );
     }
 }

components/server/src/workspace/workspace-starter.ts

@@ -419,9 +419,8 @@ export class WorkspaceStarter {
                 { retryCount: 4, retryDelay: 500 }, // We wait at most 2s until we give up, and conclude that someone else is already starting this instance
             )
             .catch((err) => {
-                if (RedisMutex.isLockError(err)) {
-                    log.warn({ instanceId }, "unable to acquire lock for workspace instance start");
-                    return;
+                if (!RedisMutex.isLockedError(err)) {
+                    log.warn({ instanceId }, "unexpected error during workspace instance start", err);
                 }
             });
     }
@@ -582,9 +581,10 @@ export class WorkspaceStarter {
                 // due to the reconciliation loop we might have already started the workspace, especially in the "pending" phase
                 const workspaceAlreadyExists = await this.existsWithWsManager(ctx, instance);
                 if (workspaceAlreadyExists) {
-                    log.warn(
+                    log.debug(
                         { instanceId: instance.id, workspaceId: instance.workspaceId },
                         "workspace already exists, not starting again",
+                        { phase: instance.status.phase },
                     );
                     return;
                 }