Merge branch '546-diagnostic-log-bloating' into 'master'

fix: stop bloating diagnostic logs (#546)

Closes #546

See merge request postgres-ai/database-lab!827

Author: NikolayS

engine/.gitlab-ci.yml

@@ -56,7 +56,7 @@ lint:
 ### Build binary.
 build-binary-alpine:
   <<: *only_engine
-  image: golang:1.18-alpine
+  image: golang:1.20-alpine
   stage: build-binary
   artifacts:
     paths:

engine/internal/diagnostic/logs.go

@@ -72,7 +72,7 @@ func CollectDiagnostics(ctx context.Context, client *client.Client, filterArgs f
 }
 
 // CollectContainerDiagnostics collect specific container diagnostics information.
-func CollectContainerDiagnostics(ctx context.Context, client *client.Client, containerName string) {
+func CollectContainerDiagnostics(ctx context.Context, client *client.Client, containerName, dbDataDir string) {
 	diagnosticsDir, err := util.GetLogsPath(time.Now().Format(timeFormat))
 
 	if err != nil {
@@ -89,6 +89,12 @@ func CollectContainerDiagnostics(ctx context.Context, client *client.Client, con
 	if err != nil {
 		log.Warn("Failed to collect container logs ", containerName, err)
 	}
+
+	err = collectPostgresLogs(ctx, client, diagnosticsDir, containerName, dbDataDir)
+
+	if err != nil {
+		log.Warn("Failed to collect Postgres logs ", containerName, err)
+	}
 }
 
 func collectContainersOutput(ctx context.Context, client *client.Client, diagnosticDir string, filterArgs filters.Args) error {

engine/internal/provision/databases/postgres/postgres.go

@@ -13,7 +13,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 
 	_ "github.com/lib/pq" // Register Postgres database driver.
@@ -61,6 +60,8 @@ func Start(r runners.Runner, c *resources.AppConfig) error {
 		return errors.Wrap(err, "failed to run container")
 	}
 
+	log.Dbg("Container has been started. Running Postgres...")
+
 	// Waiting for server to become ready and promote if needed.
 	first := true
 	cnt := 0
@@ -112,7 +113,7 @@ func Start(r runners.Runner, c *resources.AppConfig) error {
 		cnt++
 
 		if cnt > waitPostgresTimeout {
-			err := collectDiagnostics(c)
+			collectDiagnostics(c)
 
 			if runnerErr := Stop(r, c.Pool, c.CloneName); runnerErr != nil {
 				log.Err(runnerErr)
@@ -127,21 +128,13 @@ func Start(r runners.Runner, c *resources.AppConfig) error {
 	return nil
 }
 
-func collectDiagnostics(c *resources.AppConfig) error {
+func collectDiagnostics(c *resources.AppConfig) {
 	dockerClient, err := client.NewClientWithOpts(client.FromEnv)
 	if err != nil {
 		log.Fatal("Failed to create a Docker client:", err)
 	}
 
-	filterArgs := filters.NewArgs(
-		filters.KeyValuePair{Key: "label",
-			Value: fmt.Sprintf("%s=%s", docker.LabelClone, c.Pool.Name)})
-
-	if err := diagnostic.CollectDiagnostics(context.Background(), dockerClient, filterArgs, c.CloneName, c.DataDir()); err != nil {
-		log.Err("Failed to collect container diagnostics", err)
-	}
-
-	return err
+	diagnostic.CollectContainerDiagnostics(context.Background(), dockerClient, c.CloneName, c.DataDir())
 }
 
 // Stop stops Postgres instance.

engine/internal/provision/mode_local.go

@@ -31,6 +31,7 @@ import (
 	"gitlab.com/postgres-ai/database-lab/v3/internal/provision/resources"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/provision/runners"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools"
+	"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/fs"
 	"gitlab.com/postgres-ai/database-lab/v3/pkg/log"
 	"gitlab.com/postgres-ai/database-lab/v3/pkg/models"
 	"gitlab.com/postgres-ai/database-lab/v3/pkg/util"
@@ -188,6 +189,10 @@ func (p *Provisioner) StartSession(snapshotID string, user resources.EphemeralUs
 	appConfig := p.getAppConfig(fsm.Pool(), name, port)
 	appConfig.SetExtraConf(extraConfig)
 
+	if err := fs.CleanupLogsDir(appConfig.DataDir()); err != nil {
+		log.Warn("Failed to clean up logs directory:", err.Error())
+	}
+
 	if err = postgres.Start(p.runner, appConfig); err != nil {
 		return nil, errors.Wrap(err, "failed to start a container")
 	}
@@ -284,6 +289,10 @@ func (p *Provisioner) ResetSession(session *resources.Session, snapshotID string
 	appConfig := p.getAppConfig(newFSManager.Pool(), name, session.Port)
 	appConfig.SetExtraConf(session.ExtraConfig)
 
+	if err := fs.CleanupLogsDir(appConfig.DataDir()); err != nil {
+		log.Warn("Failed to clean up logs directory:", err.Error())
+	}
+
 	if err = postgres.Start(p.runner, appConfig); err != nil {
 		return nil, errors.Wrap(err, "failed to start container")
 	}

engine/internal/retrieval/engine/postgres/snapshot/physical.go

@@ -11,6 +11,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"os"
 	"path"
 	"strings"
 	"sync"
@@ -37,6 +38,7 @@ import (
 	"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/activity"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/cont"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/defaults"
+	"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/fs"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/health"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/pgtool"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/query"
@@ -65,6 +67,9 @@ const (
 	// WAL parsing constants.
 	walNameLen  = 24
 	pgVersion10 = 10
+
+	logDirName              = "log"
+	defaultLogRetentionDays = 7
 )
 
 var defaultRecoveryCfg = map[string]string{
@@ -351,6 +356,11 @@ func (p *PhysicalInitial) run(ctx context.Context) (err error) {
 		return errors.Wrapf(err, "failed to create \"pre\" clone %s", cloneName)
 	}
 
+	cloneDataDir := path.Join(p.fsPool.ClonesDir(), cloneName, p.fsPool.DataSubDir)
+	if err := fs.CleanupLogsDir(cloneDataDir); err != nil {
+		log.Warn("Failed to clean up logs directory:", err.Error())
+	}
+
 	defer func() {
 		if err != nil {
 			if errDestroy := p.cloneManager.DestroyClone(cloneName); errDestroy != nil {
@@ -361,7 +371,7 @@ func (p *PhysicalInitial) run(ctx context.Context) (err error) {
 
 	// Promotion.
 	if p.options.Promotion.Enabled {
-		if err := p.promoteInstance(ctx, path.Join(p.fsPool.ClonesDir(), cloneName, p.fsPool.DataSubDir), syState); err != nil {
+		if err := p.promoteInstance(ctx, cloneDataDir, syState); err != nil {
 			return errors.Wrap(err, "failed to promote instance")
 		}
 	}
@@ -387,6 +397,45 @@ func (p *PhysicalInitial) run(ctx context.Context) (err error) {
 
 	p.tm.SendEvent(ctx, telemetry.SnapshotCreatedEvent, telemetry.SnapshotCreated{})
 
+	if err := p.cleanupOldLogs(); err != nil {
+		log.Warn("cannot clean up old logs", err.Error())
+	}
+
+	return nil
+}
+
+func (p *PhysicalInitial) cleanupOldLogs() error {
+	lastWeekTime := time.Now().AddDate(0, 0, -1*defaultLogRetentionDays)
+
+	log.Dbg("Cleaning up PGDATA logs older than", lastWeekTime.Format(time.DateTime))
+
+	logDir := path.Join(p.fsPool.DataDir(), logDirName)
+
+	dirEntries, err := os.ReadDir(logDir)
+	if err != nil {
+		return err
+	}
+
+	var fileCounter int
+
+	for _, logFile := range dirEntries {
+		info, err := logFile.Info()
+		if err != nil {
+			continue
+		}
+
+		if info.ModTime().Before(lastWeekTime) {
+			logFilename := path.Join(logDir, logFile.Name())
+			if err := os.RemoveAll(logFilename); err != nil {
+				log.Warn("cannot remove old log file %s: %s", logFilename, err.Error())
+			}
+
+			fileCounter++
+		}
+	}
+
+	log.Dbg("Old PGDATA logs have been cleaned. Number of deleted files: ", fileCounter)
+
 	return nil
 }
 

engine/internal/retrieval/engine/postgres/tools/fs/tools.go

@@ -6,11 +6,17 @@
 package fs
 
 import (
+	"fmt"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
 )
 
+const (
+	logDirectory = "log"
+)
+
 // CopyDirectoryContent copies all files from one directory to another.
 func CopyDirectoryContent(sourceDir, dataDir string) error {
 	entries, err := os.ReadDir(sourceDir)
@@ -73,3 +79,22 @@ func AppendFile(file string, data []byte) error {
 
 	return nil
 }
+
+// CleanupLogsDir removes old log files from the clone directory.
+func CleanupLogsDir(dataDir string) error {
+	logPath := path.Join(dataDir, logDirectory)
+
+	logDir, err := os.ReadDir(logPath)
+	if err != nil {
+		return fmt.Errorf("cannot read directory %s: %v", logPath, err.Error())
+	}
+
+	for _, logFile := range logDir {
+		logName := path.Join(logPath, logFile.Name())
+		if err := os.RemoveAll(logName); err != nil {
+			return fmt.Errorf("cannot remove %s: %v", logName, err.Error())
+		}
+	}
+
+	return nil
+}

engine/internal/retrieval/engine/postgres/tools/tools.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/AlekSi/pointer"
+	"github.com/ahmetalpbalkan/dlog"
 	"github.com/docker/cli/cli/streams"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
@@ -682,7 +683,7 @@ func CopyContainerLogs(ctx context.Context, docker *client.Client, containerName
 		}
 	}()
 
-	if _, err := io.Copy(writeFile, reader); err != nil {
+	if _, err := io.Copy(writeFile, dlog.NewReader(reader)); err != nil {
 		return fmt.Errorf("failed to copy container output %w", err)
 	}