Merge branch '425-health-check-retries' into 'master'

fix(engine): adjust container readiness health checks (#425)

Closes #425

See merge request postgres-ai/database-lab!597

Author: agneum

engine/internal/retrieval/engine/postgres/tools/health/healthcheck.go

@@ -46,21 +46,21 @@ func GetConfig(username, dbname string, options ...ContainerOption) *container.H
 	return healthConfig
 }
 
-// OptionRetries allows overwrite retries counter.
+// OptionRetries allows overwriting retries counter.
 func OptionRetries(retries int) ContainerOption {
 	return func(h *container.HealthConfig) {
 		h.Retries = retries
 	}
 }
 
-// OptionInterval allows overwrite a health check interval.
+// OptionInterval allows overwriting a health check interval.
 func OptionInterval(interval time.Duration) ContainerOption {
 	return func(h *container.HealthConfig) {
 		h.Interval = interval
 	}
 }
 
-// OptionTest allows overwrite a health check test command.
+// OptionTest allows overwriting a health check test command.
 func OptionTest(testCommand string) ContainerOption {
 	return func(h *container.HealthConfig) {
 		if testCommand != "" {

engine/internal/retrieval/engine/postgres/tools/tools.go

@@ -321,6 +321,8 @@ func StopPostgres(ctx context.Context, dockerClient *client.Client, containerID,
 func CheckContainerReadiness(ctx context.Context, dockerClient *client.Client, containerID string) (err error) {
 	log.Msg("Check container readiness: ", containerID)
 
+	var errorRepeats bool
+
 	for {
 		select {
 		case <-ctx.Done():
@@ -344,11 +346,22 @@ func CheckContainerReadiness(ctx context.Context, dockerClient *client.Client, c
 			}
 
 			if healthCheckLength := len(resp.State.Health.Log); healthCheckLength > 0 {
+				// Checking exit code 2 and 3 because pg_isready returns
+				//  0 to the shell if the server is accepting connections normally,
+				//  1 if the server is rejecting connections (for example during startup),
+				//  2 if there was no response to the connection attempt, and
+				//  3 if no attempt was made (for example due to invalid parameters).
+				// Supposedly, the status 2 will be returned in cases where the server is not running
+				// and will not start on its own, so there is no reason to wait for all specified retries.
 				if lastHealthCheck := resp.State.Health.Log[healthCheckLength-1]; lastHealthCheck.ExitCode > 1 {
-					return &ErrHealthCheck{
-						ExitCode: lastHealthCheck.ExitCode,
-						Output:   lastHealthCheck.Output,
+					if errorRepeats {
+						return &ErrHealthCheck{
+							ExitCode: lastHealthCheck.ExitCode,
+							Output:   lastHealthCheck.Output,
+						}
 					}
+
+					errorRepeats = true
 				}
 			}
 		}