diff --git a/cmd/dump.go b/cmd/dump.go index 678c9a1..7a6ee20 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -118,6 +118,8 @@ func dumpCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, er if !v.IsSet("skip-extended-insert") && dumpConfig != nil && dumpConfig.SkipExtendedInsert != nil { skipExtendedInsert = *dumpConfig.SkipExtendedInsert } + includeGeneratedColumns := v.GetBool("include-generated-columns") + // Note: config file support for include-generated-columns would require updates to api.Dump compact := v.GetBool("compact") if !v.IsSet("compact") && dumpConfig != nil && dumpConfig.Compact != nil { compact = *dumpConfig.Compact @@ -251,24 +253,25 @@ func dumpCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, er defer dumpSpan.End() uid := uuid.New() dumpOpts := core.DumpOptions{ - Targets: targets, - Safechars: safechars, - DBNames: include, - DBConn: cmdConfig.dbconn, - Compressor: compressor, - Encryptor: encryptor, - Exclude: exclude, - PreBackupScripts: preBackupScripts, - PostBackupScripts: postBackupScripts, - SuppressUseDatabase: noDatabaseName, - SkipExtendedInsert: skipExtendedInsert, - Compact: compact, - Triggers: triggers, - Routines: routines, - MaxAllowedPacket: maxAllowedPacket, - Run: uid, - FilenamePattern: filenamePattern, - Parallelism: parallel, + Targets: targets, + Safechars: safechars, + DBNames: include, + DBConn: cmdConfig.dbconn, + Compressor: compressor, + Encryptor: encryptor, + Exclude: exclude, + PreBackupScripts: preBackupScripts, + PostBackupScripts: postBackupScripts, + SuppressUseDatabase: noDatabaseName, + SkipExtendedInsert: skipExtendedInsert, + Compact: compact, + Triggers: triggers, + Routines: routines, + IncludeGeneratedColumns: includeGeneratedColumns, + MaxAllowedPacket: maxAllowedPacket, + Run: uid, + FilenamePattern: filenamePattern, + Parallelism: parallel, } _, err := executor.Dump(tracerCtx, dumpOpts) if err != nil { @@ -315,6 +318,9 @@ S3: If it is a URL of the format s3://bucketname/path then it will connect via S // skip extended insert in dump; instead, one INSERT per record in each table flags.Bool("skip-extended-insert", false, "Skip extended insert in dump; instead, one INSERT per record in each table.") + // include generated columns in dump + flags.Bool("include-generated-columns", false, "Include generated columns in the dump. By default, generated and virtual columns are excluded.") + // frequency flags.Int("frequency", defaultFrequency, "how often to run backups, in minutes") diff --git a/docs/configuration.md b/docs/configuration.md index 5bf77bb..f1bb119 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -73,6 +73,7 @@ The following are the environment variables, CLI flags and configuration file op | names of databases to exclude from the dump | B | `exclude` | `DB_DUMP_EXCLUDE` | `dump.exclude` | | | do not include `USE ;` statement in the dump | B | `no-database-name` | `DB_DUMP_NO_DATABASE_NAME` | `dump.noDatabaseName` | `false` | | Replace single long INSERT statement per table with one INSERT statement per line | B | `skip-extended-insert` | `DB_DUMP_SKIP_EXTENDED_INSERT` | `dump.skipExtendedInsert` | `false` | +| Include generated columns in dump (not virtual columns) | B | `include-generated-columns` | `DB_DUMP_INCLUDE_GENERATED_COLUMNS` | `dump.includeGeneratedColumns` | `false` | | restore to a specific database | R | `restore --database` | `RESTORE_DATABASE` | `restore.database` | | | how often to do a dump or prune, in minutes | BP | `dump --frequency` | `DB_DUMP_FREQUENCY` | `dump.schedule.frequency` | `1440` (in minutes), i.e. once per day | | what time to do the first dump or prune | BP | `dump --begin` | `DB_DUMP_BEGIN` | `dump.schedule.begin` | `0`, i.e. immediately | @@ -139,6 +140,7 @@ for details of each. * `once`: boolean, run once and exit * `compression`: string, the compression to use * `compact`: boolean, compact the dump + * `includeGeneratedColumns`: boolean, include columns marked as `GENERATED` in the dump (does not include `VIRTUAL` columns), when set true, it makes the dump include the generated/default values instead of relying on the default expressions when restoring. * `triggersAndFunctions`: boolean, include triggers and functions and procedures in the dump * `maxAllowedPacket`: int, max packet size * `filenamePattern`: string, the filename pattern diff --git a/examples/configs/local.yaml b/examples/configs/local.yaml index 4c9b05b..899aaf3 100644 --- a/examples/configs/local.yaml +++ b/examples/configs/local.yaml @@ -21,6 +21,11 @@ spec: safechars: true # defaults to false noDatabaseName: false # remove the `USE ` statement from backup files, defaults to false skipExtendedInsert: false # replace single long INSERT statement per table with one INSERT statement per line, defaults to false + includeGeneratedColumns: false # include columns marked as GENERATED; VIRTUAL columns remain excluded. Defaults to false + # When enabled, columns with defaults or generated values (for example + # `create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP`) will be + # dumped as part of the INSERT rows. Example: + # INSERT INTO `mytable` (`id`, `name`, `create_time`) VALUES (1, 'alice', '2025-11-14 09:30:00'); # schedule to dump, can use one of: cron, frequency, once. If frequency is set, begin will be checked schedule: once: true # run only once and exit; ignores all other scheduling. Defaults to false diff --git a/pkg/core/dump.go b/pkg/core/dump.go index d0c8573..b8d88ed 100644 --- a/pkg/core/dump.go +++ b/pkg/core/dump.go @@ -108,14 +108,15 @@ func (e *Executor) Dump(ctx context.Context, opts DumpOptions) (DumpResults, err results.DumpStart = time.Now() dbDumpCtx, dbDumpSpan := tracer.Start(ctx, "database_dump") if err := database.Dump(dbDumpCtx, dbconn, database.DumpOpts{ - Compact: compact, - Triggers: triggers, - Routines: routines, - SuppressUseDatabase: suppressUseDatabase, - SkipExtendedInsert: skipExtendedInsert, - MaxAllowedPacket: maxAllowedPacket, - PostDumpDelay: opts.PostDumpDelay, - Parallelism: parallelism, + Compact: compact, + Triggers: triggers, + Routines: routines, + SuppressUseDatabase: suppressUseDatabase, + SkipExtendedInsert: skipExtendedInsert, + MaxAllowedPacket: maxAllowedPacket, + IncludeGeneratedColumns: opts.IncludeGeneratedColumns, + PostDumpDelay: opts.PostDumpDelay, + Parallelism: parallelism, }, dw); err != nil { dbDumpSpan.SetStatus(codes.Error, err.Error()) dbDumpSpan.End() diff --git a/pkg/core/dumpoptions.go b/pkg/core/dumpoptions.go index 408734d..afb5d19 100644 --- a/pkg/core/dumpoptions.go +++ b/pkg/core/dumpoptions.go @@ -11,23 +11,24 @@ import ( ) type DumpOptions struct { - Targets []storage.Storage - Safechars bool - DBNames []string - DBConn *database.Connection - Compressor compression.Compressor - Encryptor encrypt.Encryptor - Exclude []string - PreBackupScripts string - PostBackupScripts string - Compact bool - Triggers bool - Routines bool - SuppressUseDatabase bool - SkipExtendedInsert bool - MaxAllowedPacket int - Run uuid.UUID - FilenamePattern string + Targets []storage.Storage + Safechars bool + DBNames []string + DBConn *database.Connection + Compressor compression.Compressor + Encryptor encrypt.Encryptor + Exclude []string + PreBackupScripts string + PostBackupScripts string + Compact bool + Triggers bool + Routines bool + SuppressUseDatabase bool + SkipExtendedInsert bool + IncludeGeneratedColumns bool + MaxAllowedPacket int + Run uuid.UUID + FilenamePattern string // PostDumpDelay inafter each dump is complete, while holding connection open. Do not use outside of tests. PostDumpDelay time.Duration // Parallelism how many databases to back up at once, consuming that number of threads diff --git a/pkg/database/dump.go b/pkg/database/dump.go index 0fab972..ce0d6a8 100644 --- a/pkg/database/dump.go +++ b/pkg/database/dump.go @@ -10,12 +10,13 @@ import ( ) type DumpOpts struct { - Compact bool - Triggers bool - Routines bool - SuppressUseDatabase bool - SkipExtendedInsert bool - MaxAllowedPacket int + Compact bool + Triggers bool + Routines bool + SuppressUseDatabase bool + SkipExtendedInsert bool + MaxAllowedPacket int + IncludeGeneratedColumns bool // PostDumpDelay after each dump is complete, while holding connection open. Do not use outside of tests. PostDumpDelay time.Duration Parallelism int @@ -52,17 +53,18 @@ func Dump(ctx context.Context, dbconn *Connection, opts DumpOpts, writers []Dump defer func() { <-sem }() for _, schema := range writer.Schemas { dumper := &mysql.Data{ - Out: writer.Writer, - Connection: db, - Schema: schema, - Host: dbconn.Host, - Compact: opts.Compact, - Triggers: opts.Triggers, - Routines: opts.Routines, - SuppressUseDatabase: opts.SuppressUseDatabase, - SkipExtendedInsert: opts.SkipExtendedInsert, - MaxAllowedPacket: opts.MaxAllowedPacket, - PostDumpDelay: opts.PostDumpDelay, + Out: writer.Writer, + Connection: db, + Schema: schema, + Host: dbconn.Host, + Compact: opts.Compact, + Triggers: opts.Triggers, + Routines: opts.Routines, + SuppressUseDatabase: opts.SuppressUseDatabase, + SkipExtendedInsert: opts.SkipExtendedInsert, + MaxAllowedPacket: opts.MaxAllowedPacket, + PostDumpDelay: opts.PostDumpDelay, + IncludeGeneratedColumns: opts.IncludeGeneratedColumns, } // return on any error if err := dumper.Dump(); err != nil { diff --git a/pkg/database/mysql/dump.go b/pkg/database/mysql/dump.go index 9c93a2d..d325461 100644 --- a/pkg/database/mysql/dump.go +++ b/pkg/database/mysql/dump.go @@ -39,21 +39,22 @@ Data struct to configure dump behavior LockTables: Lock all tables for the duration of the dump */ type Data struct { - Out io.Writer - Connection *sql.DB - IgnoreTables []string - MaxAllowedPacket int - LockTables bool - Schema string - Compact bool - Triggers bool - Routines bool - Host string - SuppressUseDatabase bool - SkipExtendedInsert bool - Charset string - Collation string - PostDumpDelay time.Duration + Out io.Writer + Connection *sql.DB + IgnoreTables []string + MaxAllowedPacket int + LockTables bool + Schema string + Compact bool + Triggers bool + Routines bool + Host string + SuppressUseDatabase bool + SkipExtendedInsert bool + Charset string + Collation string + PostDumpDelay time.Duration + IncludeGeneratedColumns bool tx *sql.Tx headerTmpl *template.Template diff --git a/pkg/database/mysql/table.go b/pkg/database/mysql/table.go index 104a5e0..858f535 100644 --- a/pkg/database/mysql/table.go +++ b/pkg/database/mysql/table.go @@ -132,7 +132,17 @@ func (table *baseTable) initColumnData() error { // Ignore the virtual columns and generated columns // if there is an Extra column and it is a valid string, then only include this column if // the column is not marked as VIRTUAL or GENERATED - if !info[extraIndex].Valid || (!strings.Contains(info[extraIndex].String, "VIRTUAL") && !strings.Contains(info[extraIndex].String, "GENERATED")) { + // Unless IncludeGeneratedColumns is true, in which case include them + shouldInclude := true + if info[extraIndex].Valid { + extra := info[extraIndex].String + if strings.Contains(extra, "VIRTUAL") { + shouldInclude = false + } else if strings.Contains(extra, "GENERATED") && !table.data.IncludeGeneratedColumns { + shouldInclude = false + } + } + if shouldInclude { result = append(result, info[fieldIndex].String) } }