Added a new configuration key include-generated-columns (or environment variable DB_DUMP_INCLUDE_GENERATED_COLUMNS) to allow users to control whether generated columns should be included in database dumps.

Signed-off-by: chenen <itfunx@hotmail.com>

Author: n2em

CHANGES_GENERATED_COLUMNS.md

@@ -0,0 +1,76 @@
+# Add Configuration Option to Include Generated Columns
+
+## Overview
+Added a new configuration key `include-generated-columns` (or environment variable `DB_DUMP_INCLUDE_GENERATED_COLUMNS`) to allow users to control whether generated columns should be included in database dumps.
+
+## Problem
+Previously, the code would always exclude columns marked as `VIRTUAL` or `GENERATED` in the "Extra" field of MySQL's `SHOW COLUMNS` output. This was hardcoded behavior with no way to override it.
+
+## Solution
+Implemented a configurable option to control this behavior by:
+
+1. Adding a new `IncludeGeneratedColumns` boolean field throughout the configuration chain
+2. Modifying the column filtering logic to respect this setting
+
+## Files Modified
+
+### 1. `pkg/database/mysql/dump.go`
+- Added `IncludeGeneratedColumns bool` field to the `Data` struct
+- This field controls whether generated columns should be included in dumps
+
+### 2. `pkg/database/mysql/table.go`
+- Modified `initColumnData()` function to use `IncludeGeneratedColumns` setting
+- **Before**: Columns with `GENERATED` or `VIRTUAL` were always excluded
+- **After**: 
+  - Columns with `VIRTUAL` are always excluded (computed columns that can't be stored)
+  - Columns with `GENERATED` are excluded UNLESS `IncludeGeneratedColumns` is `true`
+
+### 3. `pkg/database/dump.go`
+- Added `IncludeGeneratedColumns bool` to `DumpOpts` struct
+- Updated the `Dump()` function to pass this setting to `mysql.Data`
+
+### 4. `pkg/core/dumpoptions.go`
+- Added `IncludeGeneratedColumns bool` to `DumpOptions` struct
+- This carries the setting from CLI/config through to the database layer
+
+### 5. `pkg/core/dump.go`
+- Updated `Dump()` function to include `IncludeGeneratedColumns` when creating `database.DumpOpts`
+
+### 6. `cmd/dump.go`
+- Added `--include-generated-columns` CLI flag (boolean, defaults to false)
+- Added parsing logic to read the flag value
+- Added `includeGeneratedColumns` variable to the command execution path
+- Passed the setting to `core.DumpOptions` struct
+
+## Usage
+
+### Command Line
+```bash
+# Enable inclusion of generated columns in dump
+db-backup dump --server myhost --target file:///backups --include-generated-columns
+
+# Or using environment variable
+export DB_DUMP_INCLUDE_GENERATED_COLUMNS=true
+db-backup dump --server myhost --target file:///backups
+```
+
+### Default Behavior
+By default (`--include-generated-columns` is not set), the behavior is unchanged:
+- `VIRTUAL` columns are excluded
+- `GENERATED` columns are excluded
+
+### With the Flag
+When `--include-generated-columns` is set:
+- `VIRTUAL` columns are still excluded (they cannot be restored)
+- `GENERATED` columns are included in the dump
+
+## Environment Variable
+The setting can also be controlled via environment variable:
+- **Variable name**: `DB_DUMP_INCLUDE_GENERATED_COLUMNS`
+- **Values**: `true` or `false`
+- **Example**: `export DB_DUMP_INCLUDE_GENERATED_COLUMNS=true`
+
+## Notes
+- Config file support would require updates to the external `api.Dump` type in the databacker/api repository
+- The change maintains backward compatibility; existing behavior is preserved when the flag is not used
+- `VIRTUAL` columns are always excluded as they cannot be dumped and restored

cmd/dump.go

@@ -118,6 +118,8 @@ func dumpCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, er
 			if !v.IsSet("skip-extended-insert") && dumpConfig != nil && dumpConfig.SkipExtendedInsert != nil {
 				skipExtendedInsert = *dumpConfig.SkipExtendedInsert
 			}
+			includeGeneratedColumns := v.GetBool("include-generated-columns")
+			// Note: config file support for include-generated-columns would require updates to api.Dump
 			compact := v.GetBool("compact")
 			if !v.IsSet("compact") && dumpConfig != nil && dumpConfig.Compact != nil {
 				compact = *dumpConfig.Compact
@@ -251,24 +253,25 @@ func dumpCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, er
 				defer dumpSpan.End()
 				uid := uuid.New()
 				dumpOpts := core.DumpOptions{
-					Targets:             targets,
-					Safechars:           safechars,
-					DBNames:             include,
-					DBConn:              cmdConfig.dbconn,
-					Compressor:          compressor,
-					Encryptor:           encryptor,
-					Exclude:             exclude,
-					PreBackupScripts:    preBackupScripts,
-					PostBackupScripts:   postBackupScripts,
-					SuppressUseDatabase: noDatabaseName,
-					SkipExtendedInsert:  skipExtendedInsert,
-					Compact:             compact,
-					Triggers:            triggers,
-					Routines:            routines,
-					MaxAllowedPacket:    maxAllowedPacket,
-					Run:                 uid,
-					FilenamePattern:     filenamePattern,
-					Parallelism:         parallel,
+					Targets:                 targets,
+					Safechars:               safechars,
+					DBNames:                 include,
+					DBConn:                  cmdConfig.dbconn,
+					Compressor:              compressor,
+					Encryptor:               encryptor,
+					Exclude:                 exclude,
+					PreBackupScripts:        preBackupScripts,
+					PostBackupScripts:       postBackupScripts,
+					SuppressUseDatabase:     noDatabaseName,
+					SkipExtendedInsert:      skipExtendedInsert,
+					Compact:                 compact,
+					Triggers:                triggers,
+					Routines:                routines,
+					IncludeGeneratedColumns: includeGeneratedColumns,
+					MaxAllowedPacket:        maxAllowedPacket,
+					Run:                     uid,
+					FilenamePattern:         filenamePattern,
+					Parallelism:             parallel,
 				}
 				_, err := executor.Dump(tracerCtx, dumpOpts)
 				if err != nil {
@@ -315,6 +318,9 @@ S3: If it is a URL of the format s3://bucketname/path then it will connect via S
 	// skip extended insert in dump; instead, one INSERT per record in each table
 	flags.Bool("skip-extended-insert", false, "Skip extended insert in dump; instead, one INSERT per record in each table.")
 
+	// include generated columns in dump
+	flags.Bool("include-generated-columns", false, "Include generated columns in the dump. By default, generated and virtual columns are excluded.")
+
 	// frequency
 	flags.Int("frequency", defaultFrequency, "how often to run backups, in minutes")
 

pkg/core/dump.go

@@ -108,14 +108,15 @@ func (e *Executor) Dump(ctx context.Context, opts DumpOptions) (DumpResults, err
 	results.DumpStart = time.Now()
 	dbDumpCtx, dbDumpSpan := tracer.Start(ctx, "database_dump")
 	if err := database.Dump(dbDumpCtx, dbconn, database.DumpOpts{
-		Compact:             compact,
-		Triggers:            triggers,
-		Routines:            routines,
-		SuppressUseDatabase: suppressUseDatabase,
-		SkipExtendedInsert:  skipExtendedInsert,
-		MaxAllowedPacket:    maxAllowedPacket,
-		PostDumpDelay:       opts.PostDumpDelay,
-		Parallelism:         parallelism,
+		Compact:                 compact,
+		Triggers:                triggers,
+		Routines:                routines,
+		SuppressUseDatabase:     suppressUseDatabase,
+		SkipExtendedInsert:      skipExtendedInsert,
+		MaxAllowedPacket:        maxAllowedPacket,
+		IncludeGeneratedColumns: opts.IncludeGeneratedColumns,
+		PostDumpDelay:           opts.PostDumpDelay,
+		Parallelism:             parallelism,
 	}, dw); err != nil {
 		dbDumpSpan.SetStatus(codes.Error, err.Error())
 		dbDumpSpan.End()

pkg/core/dumpoptions.go

@@ -25,6 +25,7 @@ type DumpOptions struct {
 	Routines            bool
 	SuppressUseDatabase bool
 	SkipExtendedInsert  bool
+	IncludeGeneratedColumns bool
 	MaxAllowedPacket    int
 	Run                 uuid.UUID
 	FilenamePattern     string

pkg/database/dump.go

@@ -10,12 +10,13 @@ import (
 )
 
 type DumpOpts struct {
-	Compact             bool
-	Triggers            bool
-	Routines            bool
-	SuppressUseDatabase bool
-	SkipExtendedInsert  bool
-	MaxAllowedPacket    int
+	Compact                 bool
+	Triggers                bool
+	Routines                bool
+	SuppressUseDatabase     bool
+	SkipExtendedInsert      bool
+	MaxAllowedPacket        int
+	IncludeGeneratedColumns bool
 	// PostDumpDelay after each dump is complete, while holding connection open. Do not use outside of tests.
 	PostDumpDelay time.Duration
 	Parallelism   int
@@ -52,17 +53,18 @@ func Dump(ctx context.Context, dbconn *Connection, opts DumpOpts, writers []Dump
 			defer func() { <-sem }()
 			for _, schema := range writer.Schemas {
 				dumper := &mysql.Data{
-					Out:                 writer.Writer,
-					Connection:          db,
-					Schema:              schema,
-					Host:                dbconn.Host,
-					Compact:             opts.Compact,
-					Triggers:            opts.Triggers,
-					Routines:            opts.Routines,
-					SuppressUseDatabase: opts.SuppressUseDatabase,
-					SkipExtendedInsert:  opts.SkipExtendedInsert,
-					MaxAllowedPacket:    opts.MaxAllowedPacket,
-					PostDumpDelay:       opts.PostDumpDelay,
+					Out:                     writer.Writer,
+					Connection:              db,
+					Schema:                  schema,
+					Host:                    dbconn.Host,
+					Compact:                 opts.Compact,
+					Triggers:                opts.Triggers,
+					Routines:                opts.Routines,
+					SuppressUseDatabase:     opts.SuppressUseDatabase,
+					SkipExtendedInsert:      opts.SkipExtendedInsert,
+					MaxAllowedPacket:        opts.MaxAllowedPacket,
+					PostDumpDelay:           opts.PostDumpDelay,
+					IncludeGeneratedColumns: opts.IncludeGeneratedColumns,
 				}
 				// return on any error
 				if err := dumper.Dump(); err != nil {

pkg/database/mysql/dump.go

@@ -39,21 +39,22 @@ Data struct to configure dump behavior
 	LockTables:       Lock all tables for the duration of the dump
 */
 type Data struct {
-	Out                 io.Writer
-	Connection          *sql.DB
-	IgnoreTables        []string
-	MaxAllowedPacket    int
-	LockTables          bool
-	Schema              string
-	Compact             bool
-	Triggers            bool
-	Routines            bool
-	Host                string
-	SuppressUseDatabase bool
-	SkipExtendedInsert  bool
-	Charset             string
-	Collation           string
-	PostDumpDelay       time.Duration
+	Out                    io.Writer
+	Connection             *sql.DB
+	IgnoreTables           []string
+	MaxAllowedPacket       int
+	LockTables             bool
+	Schema                 string
+	Compact                bool
+	Triggers               bool
+	Routines               bool
+	Host                   string
+	SuppressUseDatabase    bool
+	SkipExtendedInsert     bool
+	Charset                string
+	Collation              string
+	PostDumpDelay          time.Duration
+	IncludeGeneratedColumns bool
 
 	tx                 *sql.Tx
 	headerTmpl         *template.Template

pkg/database/mysql/table.go

@@ -132,7 +132,17 @@ func (table *baseTable) initColumnData() error {
 		// Ignore the virtual columns and generated columns
 		// if there is an Extra column and it is a valid string, then only include this column if
 		// the column is not marked as VIRTUAL or GENERATED
-		if !info[extraIndex].Valid || (!strings.Contains(info[extraIndex].String, "VIRTUAL") && !strings.Contains(info[extraIndex].String, "GENERATED")) {
+		// Unless IncludeGeneratedColumns is true, in which case include them
+		shouldInclude := true
+		if info[extraIndex].Valid {
+			extra := info[extraIndex].String
+			if strings.Contains(extra, "VIRTUAL") {
+				shouldInclude = false
+			} else if strings.Contains(extra, "GENERATED") && !table.data.IncludeGeneratedColumns {
+				shouldInclude = false
+			}
+		}
+		if shouldInclude {
 			result = append(result, info[fieldIndex].String)
 		}
 	}