From bd9f198863c3d928d8d2f13cf03c51302d40fb0e Mon Sep 17 00:00:00 2001 From: chenen Date: Fri, 14 Nov 2025 18:28:32 +0800 Subject: [PATCH 1/2] fix: Restore fails with bufio.Scanner: token too long when importing large dump files Signed-off-by: chenen --- pkg/database/restore.go | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/pkg/database/restore.go b/pkg/database/restore.go index 37a9c20..9fe374e 100644 --- a/pkg/database/restore.go +++ b/pkg/database/restore.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "regexp" + "strings" ) const ( @@ -31,20 +32,33 @@ func Restore(ctx context.Context, dbconn *Connection, databasesMap map[string]st if err != nil { return fmt.Errorf("failed to restore database: %w", err) } - scanBuf := []byte{} - scanner := bufio.NewScanner(r) - // increase the buffer size - scanner.Buffer(scanBuf, defaultMaxAllowedPacket) //TODO should be a configurable option like with dump + reader := bufio.NewReader(r) var current string - for scanner.Scan() { - line := scanner.Text() + for { + line, err := reader.ReadString('\n') + if err != nil && err != io.EOF { + _ = tx.Rollback() + return fmt.Errorf("failed to restore database: %w", err) + } + // strip CRLF/newline + line = strings.TrimRight(line, "\r\n") if line == "" { + if err == io.EOF { + break + } continue } current += line + "\n" + + // if the line does not end with a semicolon, keep accumulating if line[len(line)-1] != ';' { + if err == io.EOF { + // EOF reached but statement not terminated; we'll try to execute below + break + } continue } + // if we have the line that sets the database, and we need to replace, replace it if createRegex.MatchString(current) { dbName := createRegex.FindStringSubmatch(current)[3] @@ -64,9 +78,17 @@ func Restore(ctx context.Context, dbconn *Connection, databasesMap map[string]st return fmt.Errorf("failed to restore database: %w", err) } current = "" + + if err == io.EOF { + break + } } - if err := scanner.Err(); err != nil { - return fmt.Errorf("failed to restore database: %w", err) + // if there's any leftover SQL (for example last statement without newline), execute it + if strings.TrimSpace(current) != "" { + if _, err := tx.Exec(current); err != nil { + _ = tx.Rollback() + return fmt.Errorf("failed to restore database: %w", err) + } } if err := tx.Commit(); err != nil { return fmt.Errorf("failed to restore database: %w", err) From fd1bca4efd537e2aba52b4dfe01bee2126d65c5c Mon Sep 17 00:00:00 2001 From: n2em Date: Tue, 18 Nov 2025 10:57:29 +0800 Subject: [PATCH 2/2] fix: lint error, remove default max allowed packet constant Signed-off-by: chenen --- pkg/database/restore.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/database/restore.go b/pkg/database/restore.go index 9fe374e..45df73e 100644 --- a/pkg/database/restore.go +++ b/pkg/database/restore.go @@ -10,11 +10,6 @@ import ( "strings" ) -const ( - // used to define default max buffer size Scanner, counter part of dump - defaultMaxAllowedPacket = 4194304 -) - var ( useRegex = regexp.MustCompile(`(?i)^(USE\s*` + "`" + `)([^\s]+)(` + "`" + `\s*;)$`) createRegex = regexp.MustCompile(`(?i)^(CREATE\s+DATABASE\s*(\/\*.*\*\/\s*)?` + "`" + `)([^\s]+)(` + "`" + `\s*(\s*\/\*.*\*\/\s*)?\s*;$)`)