fix(engine): wait for control containers to finish stopping when DLE shutdowns (#387)

* use contexts without timeout to stop stateful control containers
* use a default timeout to stop internal server and stateless auxiliary containers

Author: agneum

engine/cmd/database-lab/main.go

@@ -126,10 +126,7 @@ func main() {
 	emergencyShutdown := func() {
 		cancel()
 
-		shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), shutdownTimeout)
-		defer shutdownCancel()
-
-		shutdownDatabaseLabEngine(shutdownCtx, docker, engProps, pm.First())
+		shutdownDatabaseLabEngine(context.Background(), docker, engProps, pm.First())
 	}
 
 	cloningSvc := cloning.NewBase(&cfg.Cloning, provisioner, tm, observingChan)
@@ -185,16 +182,18 @@ func main() {
 	<-shutdownCh
 	cancel()
 
-	shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), shutdownTimeout)
+	ctxBackground := context.Background()
+
+	shutdownCtx, shutdownCancel := context.WithTimeout(ctxBackground, shutdownTimeout)
 	defer shutdownCancel()
 
 	if err := server.Shutdown(shutdownCtx); err != nil {
 		log.Msg(err)
 	}
 
-	shutdownDatabaseLabEngine(shutdownCtx, docker, engProps, pm.First())
+	shutdownDatabaseLabEngine(ctxBackground, docker, engProps, pm.First())
 	cloningSvc.SaveClonesState()
-	tm.SendEvent(context.Background(), telemetry.EngineStoppedEvent, telemetry.EngineStopped{Uptime: server.Uptime()})
+	tm.SendEvent(ctxBackground, telemetry.EngineStoppedEvent, telemetry.EngineStopped{Uptime: server.Uptime()})
 }
 
 func getEngineProperties(ctx context.Context, dockerCLI *client.Client, cfg *config.Config) (global.EngineProps, error) {

engine/internal/retrieval/engine/postgres/tools/cont/container.go

@@ -14,7 +14,7 @@ import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
-	"github.com/docker/go-units"
+	units "github.com/docker/go-units"
 	"github.com/pkg/errors"
 
 	"gitlab.com/postgres-ai/database-lab/v3/internal/provision/pool"
@@ -87,8 +87,6 @@ func StopControlContainers(ctx context.Context, dockerClient *client.Client, ins
 			if err := tools.StopPostgres(ctx, dockerClient, controlCont.ID, fsm.Pool().DataDir(), tools.DefaultStopTimeout); err != nil {
 				log.Msg("Failed to stop Postgres", err)
 				tools.PrintContainerLogs(ctx, dockerClient, controlCont.ID)
-
-				continue
 			}
 		}
 
@@ -114,7 +112,11 @@ func CleanUpControlContainers(ctx context.Context, dockerClient *client.Client,
 // CleanUpSatelliteContainers removes satellite containers run by Database Lab Engine.
 func CleanUpSatelliteContainers(ctx context.Context, dockerClient *client.Client, instanceID string) error {
 	log.Msg("Clean up satellite containers")
-	return cleanUpContainers(ctx, dockerClient, instanceID, getSatelliteContainerFilters())
+
+	shutdownCtx, shutdownCancel := context.WithTimeout(ctx, StopTimeout)
+	defer shutdownCancel()
+
+	return cleanUpContainers(shutdownCtx, dockerClient, instanceID, getSatelliteContainerFilters())
 }
 
 // cleanUpContainers removes containers run by Database Lab Engine.