Merge branch '425-test-source-databases' into 'master'

fix(engine): verify connection and content (extensions, locales) of all databases mentioned in the list to restore (#425)

Closes #425

See merge request postgres-ai/database-lab!596

Author: agneum

engine/internal/retrieval/engine/postgres/logical/dump_default.go

@@ -6,7 +6,8 @@ package logical
 
 import (
 	"context"
-	"fmt"
+
+	"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/db"
 )
 
 type defaultDumper struct {
@@ -27,6 +28,5 @@ func (d *defaultDumper) SetConnectionOptions(_ context.Context, c *Connection) e
 }
 
 func (d *defaultDumper) GetDatabaseListQuery(username string) string {
-	return fmt.Sprintf(`select datname from pg_catalog.pg_database 
-	where not datistemplate and has_database_privilege('%s', datname, 'CONNECT')`, username)
+	return db.GetDatabaseListQuery(username)
 }

engine/internal/retrieval/engine/postgres/tools/db/pg.go

@@ -18,15 +18,24 @@ import (
 	"gitlab.com/postgres-ai/database-lab/v3/pkg/models"
 )
 
-// ConnectionString builds PostgreSQL connection string.
-func ConnectionString(host, port, username, dbname, password string) string {
-	return fmt.Sprintf("host=%s port=%s user='%s' database='%s' password='%s'", host, port, username, dbname, password)
-}
-
 const (
 	availableExtensions = "select name, default_version, coalesce(installed_version,'') from pg_available_extensions " +
 		"where installed_version is not null"
+
 	availableLocales = "select datname, lower(datcollate), lower(datctype) from pg_catalog.pg_database"
+
+	availableDBsTemplate = `select datname from pg_catalog.pg_database 
+	where not datistemplate and has_database_privilege('%s', datname, 'CONNECT')`
+
+	// maxNumberVerifiedDBs defines the maximum number of databases to verify availability as a database source.
+	// The DB source instance can contain a large number of databases, so the verification will take a long time.
+	// Therefore, we introduced a limit on the maximum number of databases to check for suitability as a source.
+	maxNumberVerifiedDBs = 5
+)
+
+var (
+	errExtensionWarning = errors.New("extension warning found")
+	errLocaleWarning    = errors.New("locale warning found")
 )
 
 type extension struct {
@@ -41,37 +50,125 @@ type locale struct {
 	ctype   string
 }
 
+// ConnectionString builds PostgreSQL connection string.
+func ConnectionString(host, port, username, dbname, password string) string {
+	return fmt.Sprintf("host=%s port=%s user='%s' database='%s' password='%s'", host, port, username, dbname, password)
+}
+
+// GetDatabaseListQuery provides the query to get the list of databases available for user.
+func GetDatabaseListQuery(username string) string {
+	return fmt.Sprintf(availableDBsTemplate, username)
+}
+
 // CheckSource checks the readiness of the source database to dump and restore processes.
 func CheckSource(ctx context.Context, conf *models.ConnectionTest, imageContent *ImageContent) (*models.TestConnection, error) {
-	connStr := ConnectionString(conf.Host, conf.Port, conf.Username, conf.DBName, conf.Password)
+	conn, tcResponse := checkConnection(ctx, conf, conf.DBName)
+	if tcResponse != nil {
+		return tcResponse, nil
+	}
+
+	defer func() {
+		if err := conn.Close(ctx); err != nil {
+			log.Dbg("failed to close connection:", err)
+		}
+	}()
+
+	dbList := conf.DBList
+
+	if len(dbList) == 0 {
+		dbSourceList, err := getDBList(ctx, conn, conf.Username)
+		if err != nil {
+			return nil, err
+		}
+
+		dbList = dbSourceList
+	}
+
+	if len(dbList) > maxNumberVerifiedDBs {
+		dbList = dbList[:maxNumberVerifiedDBs]
+		tcResponse = &models.TestConnection{
+			Status: models.TCStatusNotice,
+			Result: models.TCResultUnverifiedDB,
+			Message: "Too many databases are supposed to be checked. Only the following databases have been verified: " +
+				strings.Join(dbList, ", "),
+		}
+	}
+
+	for _, dbName := range dbList {
+		dbConn, listTC := checkConnection(ctx, conf, dbName)
+		if listTC != nil {
+			return listTC, nil
+		}
+
+		listTC, err := checkContent(ctx, dbConn, dbName, imageContent)
+		if err != nil {
+			return nil, err
+		}
+
+		if listTC != nil {
+			return listTC, nil
+		}
+	}
+
+	if tcResponse != nil {
+		return tcResponse, nil
+	}
+
+	return &models.TestConnection{
+		Status:  models.TCStatusOK,
+		Result:  models.TCResultOK,
+		Message: models.TCMessageOK,
+	}, nil
+}
+
+func getDBList(ctx context.Context, conn *pgx.Conn, dbUsername string) ([]string, error) {
+	dbList := make([]string, 0)
+
+	rows, err := conn.Query(ctx, GetDatabaseListQuery(dbUsername))
+	if err != nil {
+		return nil, fmt.Errorf("failed to perform query listing databases: %w", err)
+	}
+
+	for rows.Next() {
+		var dbName string
+		if err := rows.Scan(&dbName); err != nil {
+			return nil, fmt.Errorf("failed to scan next row in database list result set: %w", err)
+		}
+
+		dbList = append(dbList, dbName)
+	}
+
+	return dbList, nil
+}
+
+func checkConnection(ctx context.Context, conf *models.ConnectionTest, dbName string) (*pgx.Conn, *models.TestConnection) {
+	connStr := ConnectionString(conf.Host, conf.Port, conf.Username, dbName, conf.Password)
 
 	conn, err := pgx.Connect(ctx, connStr)
 	if err != nil {
 		log.Dbg("failed to test database connection:", err)
 
-		return &models.TestConnection{
+		return nil, &models.TestConnection{
 			Status:  models.TCStatusError,
 			Result:  models.TCResultConnectionError,
 			Message: err.Error(),
-		}, nil
-	}
-
-	defer func() {
-		if err := conn.Close(ctx); err != nil {
-			log.Dbg("failed to close connection:", err)
 		}
-	}()
+	}
 
 	var one int
 
 	if err := conn.QueryRow(ctx, "select 1").Scan(&one); err != nil {
-		return &models.TestConnection{
+		return nil, &models.TestConnection{
 			Status:  models.TCStatusError,
 			Result:  models.TCResultConnectionError,
 			Message: err.Error(),
-		}, nil
+		}
 	}
 
+	return conn, nil
+}
+
+func checkContent(ctx context.Context, conn *pgx.Conn, dbName string, imageContent *ImageContent) (*models.TestConnection, error) {
 	if !imageContent.IsReady() {
 		return &models.TestConnection{
 			Status: models.TCStatusNotice,
@@ -82,26 +179,30 @@ func CheckSource(ctx context.Context, conf *models.ConnectionTest, imageContent
 	}
 
 	if missing, unsupported, err := checkExtensions(ctx, conn, imageContent.Extensions()); err != nil {
+		if err != errExtensionWarning {
+			return nil, fmt.Errorf("failed to check database extensions: %w", err)
+		}
+
 		return &models.TestConnection{
 			Status:  models.TCStatusWarning,
 			Result:  models.TCResultMissingExtension,
-			Message: buildExtensionsWarningMessage(missing, unsupported),
+			Message: buildExtensionsWarningMessage(dbName, missing, unsupported),
 		}, nil
 	}
 
 	if missing, err := checkLocales(ctx, conn, imageContent.Locales(), imageContent.Databases()); err != nil {
+		if err != errLocaleWarning {
+			return nil, fmt.Errorf("failed to check database locales: %w", err)
+		}
+
 		return &models.TestConnection{
 			Status:  models.TCStatusWarning,
 			Result:  models.TCResultMissingLocale,
-			Message: buildLocalesWarningMessage(missing),
+			Message: buildLocalesWarningMessage(dbName, missing),
 		}, nil
 	}
 
-	return &models.TestConnection{
-		Status:  models.TCStatusOK,
-		Result:  models.TCResultOK,
-		Message: models.TCMessageOK,
-	}, nil
+	return nil, nil
 }
 
 func checkExtensions(ctx context.Context, conn *pgx.Conn, imageExtensions map[string]string) ([]extension, []extension, error) {
@@ -140,7 +241,7 @@ func checkExtensions(ctx context.Context, conn *pgx.Conn, imageExtensions map[st
 	}
 
 	if len(missingExtensions) != 0 || len(unsupportedVersions) != 0 {
-		return missingExtensions, unsupportedVersions, errors.New("extension warning found")
+		return missingExtensions, unsupportedVersions, errExtensionWarning
 	}
 
 	return nil, nil, nil
@@ -158,19 +259,19 @@ func toCanonicalSemver(v string) string {
 	return v
 }
 
-func buildExtensionsWarningMessage(missingExtensions, unsupportedVersions []extension) string {
+func buildExtensionsWarningMessage(dbName string, missingExtensions, unsupportedVersions []extension) string {
 	sb := &strings.Builder{}
 
 	if len(missingExtensions) > 0 {
-		sb.WriteString("There are missing extensions:")
+		sb.WriteString("There are missing extensions in the \"" + dbName + "\" database:")
 
 		formatExtensionList(sb, missingExtensions)
 
-		sb.WriteRune('\n')
+		sb.WriteString(".\n")
 	}
 
 	if len(unsupportedVersions) > 0 {
-		sb.WriteString("There are extensions with an unsupported version:")
+		sb.WriteString("There are extensions with an unsupported version in the \"" + dbName + "\" database:")
 
 		formatExtensionList(sb, unsupportedVersions)
 	}
@@ -225,17 +326,17 @@ func checkLocales(ctx context.Context, conn *pgx.Conn, imageLocales, databases m
 	}
 
 	if len(missingLocales) != 0 {
-		return missingLocales, errors.New("locale warning found")
+		return missingLocales, errLocaleWarning
 	}
 
 	return nil, nil
 }
 
-func buildLocalesWarningMessage(missingLocales []locale) string {
+func buildLocalesWarningMessage(dbName string, missingLocales []locale) string {
 	sb := &strings.Builder{}
 
 	if length := len(missingLocales); length > 0 {
-		sb.WriteString("There are missing locales:")
+		sb.WriteString("There are missing locales in the \"" + dbName + "\" database:")
 
 		for i, missing := range missingLocales {
 			sb.WriteString(fmt.Sprintf(" '%s' (collate: %s, ctype: %s)", missing.name, missing.collate, missing.ctype))

engine/pkg/models/admin.go

@@ -28,6 +28,9 @@ const (
 	// TCResultMissingLocale defines the warning about a missing locale.
 	TCResultMissingLocale = "missing_locale"
 
+	// TCResultUnverifiedDB defines notification of the presence of unverified databases.
+	TCResultUnverifiedDB = "unverified_database"
+
 	// TCMessageOK defines the source database is ready for dump and restore.
 	TCMessageOK = "Database ready for dump and restore"
 )

engine/pkg/models/configuration.go

@@ -2,11 +2,12 @@ package models
 
 // ConnectionTest defines a connection test model.
 type ConnectionTest struct {
-	Host     string `json:"host"`
-	Port     string `json:"port"`
-	DBName   string `json:"dbname"`
-	Username string `json:"username"`
-	Password string `json:"password"`
+	Host     string   `json:"host"`
+	Port     string   `json:"port"`
+	DBName   string   `json:"dbname"`
+	Username string   `json:"username"`
+	Password string   `json:"password"`
+	DBList   []string `json:"db_list"`
 }
 
 // ConfigProjection is a projection of the configuration.

ui/packages/ce/src/api/configs/testDbSource.ts

@@ -10,6 +10,7 @@ export const testDbSource = async (req: dbSource) => {
       dbname: req.dbname,
       username: req.username,
       password: req.password,
+      db_list: req.db_list
     }),
   })
 

ui/packages/shared/pages/Configuration/index.tsx

@@ -103,8 +103,8 @@ export const Configuration = observer(
 
     const onTestConnectionClick = async () => {
       setConnectionResponse(null)
-      Object.keys(connectionData).map(function (key) {
-        if (key !== 'password') {
+      Object.keys(connectionData).map(function (key: string) {
+        if (key !== 'password' && key !== 'db_list') {
           formik.validateField(key)
         }
       })

ui/packages/shared/pages/Configuration/useForm.ts

@@ -55,12 +55,26 @@ export const useForm = (onSubmit: (values: FormValues) => void) => {
     validateOnChange: false,
   })
 
+  const formatDatabaseArray = (database: any) => {
+    let databases = []
+    const splitDatabaseArray = database.split(/[,(\s)(\n)(\r)(\t)(\r\n)]/)
+
+    for (let i = 0; i < splitDatabaseArray.length; i++) {
+      if (splitDatabaseArray[i] !== '') {
+        databases.push(splitDatabaseArray[i])
+      }
+    }
+
+    return databases
+  }
+
   const connectionData = {
     host: formik.values.host,
     port: formik.values.port,
     username: formik.values.username,
     password: formik.values.password,
     dbname: formik.values.dbname,
+    ...(formik.values.databases && { db_list: formatDatabaseArray(formik.values.databases)}),
   }
 
   const isConnectionDataValid =

ui/packages/shared/pages/Instance/Info/Retrieval/RetrievalTable/index.tsx

@@ -25,8 +25,8 @@ export const RetrievalTable = ({
       </TableHead>
       <TableBody className={styles.tableBody}>
         {data && data.length > 0 ? (
-          data.map((item) => (
-            <div>
+          data.map((item, index) => (
+            <div key={index}>
               {Object.entries(item).map((val, index) => (
                 <TableRow key={index} hover className={styles.tableRow}>
                   <TableCell>