Merge branch '352-logicaldump-exclude-tables' into 'master'

feat: add excludeTables to logical dump (#348)

Closes #352 and #348

See merge request postgres-ai/database-lab!536

Author: agneum

engine/configs/config.example.logical_generic.yml

@@ -219,9 +219,15 @@ retrieval:
         # Do not specify the databases section to take all databases.
         # databases:
         #   database1:
-        #     # Option for a partial dump. Do not specify the tables section to dump all available tables.
+        # Options for a partial dump.
+        # Do not specify the tables section to dump all available tables.
+        # Corresponds to the --table option of pg_dump.
         #     tables:
         #       - table1
+        # Do not dump data for any of the tables matching pattern.
+        # Corresponds to the --exclude-table option of pg_dump.
+        #     excludeTables:
+        #       - table2
         #   database2:
         #   databaseN:
 

engine/configs/config.example.logical_rds_iam.yml

@@ -281,9 +281,15 @@ retrieval:
         #     format: directory
         #     # Compression (only for plain-text dumps): "gzip", "bzip2", or "no". Default: "no".
         #     compression: no
-        #     # Option for a partial restore. Do not specify the tables section to restore all available tables.
+        # Options for a partial dump.
+        # Do not specify the tables section to dump all available tables.
+        # Corresponds to the --table option of pg_dump.
         #     tables:
         #       - table1
+        # Do not dump data for any of the tables matching pattern.
+        # Corresponds to the --exclude-table option of pg_dump.
+        #     excludeTables:
+        #       - table2
         #   database2:
         #   databaseN:
 

engine/internal/retrieval/engine/postgres/logical/dump.go

@@ -98,10 +98,11 @@ type Source struct {
 
 // DumpDefinition describes a database for dumping.
 type DumpDefinition struct {
-	Tables      []string        `yaml:"tables"`
-	Format      string          `yaml:"format"`
-	Compression compressionType `yaml:"compression"`
-	dbName      string
+	Tables        []string        `yaml:"tables"`
+	ExcludeTables []string        `yaml:"excludeTables"`
+	Format        string          `yaml:"format"`
+	Compression   compressionType `yaml:"compression"`
+	dbName        string
 }
 
 type dumpJobConfig struct {
@@ -435,13 +436,23 @@ func (d *DumpJob) cleanupDumpLocation(ctx context.Context, dumpContID string, db
 }
 
 func (d *DumpJob) dumpDatabase(ctx context.Context, dumpContID, dbName string, dumpDefinition DumpDefinition) error {
-	dumpCommand := d.buildLogicalDumpCommand(dbName, dumpDefinition.Tables)
-	log.Msg("Running dump command: ", dumpCommand)
+	dumpCommand := d.buildLogicalDumpCommand(dbName, dumpDefinition)
+
+	if len(dumpDefinition.Tables) > 0 ||
+		len(dumpDefinition.ExcludeTables) > 0 {
+		log.Msg("Partial dump")
+
+		if len(dumpDefinition.Tables) > 0 {
+			log.Msg("Including tables: ", strings.Join(dumpDefinition.Tables, ", "))
+		}
 
-	if len(dumpDefinition.Tables) > 0 {
-		log.Msg("Partial dump will be run. Tables for dumping: ", strings.Join(dumpDefinition.Tables, ", "))
+		if len(dumpDefinition.ExcludeTables) > 0 {
+			log.Msg("Excluding tables: ", strings.Join(dumpDefinition.ExcludeTables, ", "))
+		}
 	}
 
+	log.Msg("Running dump command: ", dumpCommand)
+
 	if output, err := d.performDumpCommand(ctx, dumpContID, types.ExecConfig{
 		Tty: true,
 		Cmd: dumpCommand,
@@ -488,7 +499,12 @@ func setupPGData(ctx context.Context, dockerClient *client.Client, dataDir strin
 	return nil
 }
 
-func updateConfigs(ctx context.Context, dockerClient *client.Client, dataDir, contID string, configs map[string]string) error {
+func updateConfigs(
+	ctx context.Context,
+	dockerClient *client.Client,
+	dataDir, contID string,
+	configs map[string]string,
+) error {
 	log.Dbg("Stopping container to update configuration")
 
 	tools.StopContainer(ctx, dockerClient, contID, cont.StopTimeout)
@@ -605,21 +621,38 @@ func (d *DumpJob) getExecEnvironmentVariables() []string {
 	return execEnvs
 }
 
-func (d *DumpJob) buildLogicalDumpCommand(dbName string, tables []string) []string {
-	optionalArgs := map[string]string{
-		"--host":     d.config.db.Host,
-		"--port":     strconv.Itoa(d.config.db.Port),
-		"--username": d.config.db.Username,
-		"--dbname":   dbName,
-		"--jobs":     strconv.Itoa(d.DumpOptions.ParallelJobs),
+func (d *DumpJob) buildLogicalDumpCommand(dbName string, dump DumpDefinition) []string {
+	// don't use map here, it creates inconsistency in the order of arguments
+	dumpCmd := []string{"pg_dump", "--create"}
+
+	if d.config.db.Host != "" {
+		dumpCmd = append(dumpCmd, "--host", d.config.db.Host)
+	}
+
+	if d.config.db.Port > 0 {
+		dumpCmd = append(dumpCmd, "--port", strconv.Itoa(d.config.db.Port))
 	}
 
-	dumpCmd := append([]string{"pg_dump", "--create"}, prepareCmdOptions(optionalArgs)...)
+	if d.config.db.Username != "" {
+		dumpCmd = append(dumpCmd, "--username", d.config.db.Username)
+	}
 
-	for _, table := range tables {
+	if dbName != "" {
+		dumpCmd = append(dumpCmd, "--dbname", dbName)
+	}
+
+	if d.DumpOptions.ParallelJobs > 0 {
+		dumpCmd = append(dumpCmd, "--jobs", strconv.Itoa(d.DumpOptions.ParallelJobs))
+	}
+
+	for _, table := range dump.Tables {
 		dumpCmd = append(dumpCmd, "--table", table)
 	}
 
+	for _, table := range dump.ExcludeTables {
+		dumpCmd = append(dumpCmd, "--exclude-table", table)
+	}
+
 	// Define if restore directly or export to dump location.
 	if d.DumpOptions.Restore.Enabled {
 		dumpCmd = append(dumpCmd, "--format", customFormat)
@@ -652,18 +685,6 @@ func (d *DumpJob) buildLogicalRestoreCommand(dbName string) []string {
 	return restoreCmd
 }
 
-func prepareCmdOptions(options map[string]string) []string {
-	cmdOptions := []string{}
-
-	for optionKey, optionValue := range options {
-		if optionValue != "" {
-			cmdOptions = append(cmdOptions, optionKey, optionValue)
-		}
-	}
-
-	return cmdOptions
-}
-
 func (d *DumpJob) markDatabaseData() error {
 	if err := d.dbMarker.CreateConfig(); err != nil {
 		return errors.Wrap(err, "failed to create a DBMarker config of the database")

engine/internal/retrieval/engine/postgres/logical/restore_test.go

@@ -122,6 +122,50 @@ func TestRestoreCommandBuilding(t *testing.T) {
 	}
 }
 
+func TestDumpCommandBuilding(t *testing.T) {
+	logicalJob := &DumpJob{
+		config: dumpJobConfig{
+			db: Connection{
+				Host:     "localhost",
+				Port:     5432,
+				DBName:   "postgres",
+				Username: "john",
+				Password: "secret",
+			},
+		},
+	}
+
+	testCases := []struct {
+		copyOptions DumpOptions
+		command     []string
+	}{
+		{
+			copyOptions: DumpOptions{
+				ParallelJobs: 1,
+				DumpLocation: "/tmp/db.dump",
+				Databases: map[string]DumpDefinition{
+					"testDB": {
+						Tables: []string{"test", "users"},
+						ExcludeTables: []string{
+							"test2",
+							"users2",
+						},
+					},
+				},
+			},
+			command: []string{"pg_dump", "--create", "--host", "localhost", "--port", "5432", "--username", "john", "--dbname", "testDB", "--jobs", "1", "--table", "test", "--table", "users", "--exclude-table", "test2", "--exclude-table", "users2", "--format", "directory", "--file", "/tmp/db.dump/testDB"},
+		},
+	}
+
+	for _, tc := range testCases {
+		logicalJob.DumpOptions = tc.copyOptions
+		for dbName, definition := range tc.copyOptions.Databases {
+			dumpCommand := logicalJob.buildLogicalDumpCommand(dbName, definition)
+			assert.Equal(t, tc.command, dumpCommand)
+		}
+	}
+}
+
 func TestDiscoverDumpDirectories(t *testing.T) {
 	t.Skip("docker client is required")