improve retry for upgrade operations

Co-authored-by: mirkoCrobu <mirkocrobu@NB-0531.localdomain>

Author: mirkoCrobu

internal/orchestrator/system.go

@@ -76,7 +76,7 @@ func pullImage(ctx context.Context, stdout io.Writer, docker dockerClient.APICli
 		}
 		allErr = errors.Join(allErr, lastErr)
 
-		if !strings.Contains(lastErr.Error(), "toomanyrequests") {
+		if !isTemporaryDockerError(lastErr) {
 			return allErr // Non-retryable error
 		}
 
@@ -119,6 +119,21 @@ func pullImage(ctx context.Context, stdout io.Writer, docker dockerClient.APICli
 	}
 	return nil
 }
+func isTemporaryDockerError(err error) bool {
+	errorString := err.Error()
+	transientSubstrings := []string{
+		"toomanyrequests",
+		"Client.Timeout exceeded",
+		"request canceled while waiting for connection",
+	}
+
+	for _, sub := range transientSubstrings {
+		if strings.Contains(errorString, sub) {
+			return true
+		}
+	}
+	return false
+}
 
 // List of prefixes used to identify current or past Arduino images. Used both during 'system init' and during cleanup.
 var imagePrefixes = []string{"ghcr.io/bcmi-labs/", "public.ecr.aws/arduino/", "ghcr.io/arduino/", "influxdb"}

internal/update/apt/service.go

@@ -176,7 +176,15 @@ func runUpdateCommand(ctx context.Context) error {
 
 func runUpgradeCommand(ctx context.Context, names []string) iter.Seq2[string, error] {
 	env := []string{"NEEDRESTART_MODE=l"}
-	args := append([]string{"sudo", "apt-get", "install", "--only-upgrade", "-y"}, names...)
+
+	aptOptions := []string{
+		"-o", "Acquire::Retries=3",
+		"-o", "Acquire::http::Timeout=30",
+		"-o", "Acquire::https::Timeout=30",
+	}
+	args := []string{"sudo", "apt-get", "install", "--only-upgrade", "-y"}
+	args = append(args, aptOptions...)
+	args = append(args, names...)
 
 	return func(yield func(string, error) bool) {
 		upgradeCmd, err := paths.NewProcess(env, args...)