Merge branch 'master' of https://git.postgrespro.ru/pgpro-dev/pg_probackup into testgres1

Conflicts:
	tests/auth_test.py
	tests/helpers/ptrack_helpers.py
	tests/ptrack.py

Author: s.logvinenko

src/backup.c

@@ -394,6 +394,8 @@ pgBackupWriteControl(FILE *out, pgBackup *backup)
 	fprintf(out, "block-size = %u\n", backup->block_size);
 	fprintf(out, "xlog-block-size = %u\n", backup->wal_block_size);
 	fprintf(out, "checksum-version = %u\n", backup->checksum_version);
+	if (backup->server_version[0] != '\0')
+		fprintf(out, "server-version = %s\n", backup->server_version);
 
 	fprintf(out, "\n#Result backup info\n");
 	fprintf(out, "timelineid = %d\n", backup->tli);
@@ -475,6 +477,7 @@ readBackupControlFile(const char *path)
 	char	   *status = NULL;
 	char	   *parent_backup = NULL;
 	char	   *compress_alg = NULL;
+	char	   *server_version = NULL;
 	int		   *compress_level;
 	bool	   *from_replica;
 
@@ -492,7 +495,8 @@ readBackupControlFile(const char *path)
 		{'I', 0, "wal-bytes",			&backup->wal_bytes, SOURCE_FILE_STRICT},
 		{'u', 0, "block-size",			&backup->block_size, SOURCE_FILE_STRICT},
 		{'u', 0, "xlog-block-size",		&backup->wal_block_size, SOURCE_FILE_STRICT},
-		{'u', 0, "checksum_version",	&backup->checksum_version, SOURCE_FILE_STRICT},
+		{'u', 0, "checksum-version",	&backup->checksum_version, SOURCE_FILE_STRICT},
+		{'s', 0, "server-version",		&server_version, SOURCE_FILE_STRICT},
 		{'b', 0, "stream",				&backup->stream, SOURCE_FILE_STRICT},
 		{'s', 0, "status",				&status, SOURCE_FILE_STRICT},
 		{'s', 0, "parent-backup-id",	&parent_backup, SOURCE_FILE_STRICT},
@@ -565,6 +569,13 @@ readBackupControlFile(const char *path)
 		free(parent_backup);
 	}
 
+	if (server_version)
+	{
+		StrNCpy(backup->server_version, server_version,
+				sizeof(backup->server_version));
+		pfree(server_version);
+	}
+
 	return backup;
 }
 
@@ -591,6 +602,24 @@ parse_backup_mode(const char *value)
 	return BACKUP_MODE_INVALID;
 }
 
+const char *
+deparse_backup_mode(BackupMode mode)
+{
+	switch (mode)
+	{
+		case BACKUP_MODE_FULL:
+			return "full";
+		case BACKUP_MODE_DIFF_PAGE:
+			return "page";
+		case BACKUP_MODE_DIFF_PTRACK:
+			return "ptrack";
+		case BACKUP_MODE_INVALID:
+			return "invalid";
+	}
+
+	return NULL;
+}
+
 /* free pgBackup object */
 void
 pgBackupFree(void *backup)

src/catalog.c

@@ -9,7 +9,8 @@
 
 #include "pg_probackup.h"
 
-static void opt_log_level(pgut_option *opt, const char *arg);
+static void opt_log_level_console(pgut_option *opt, const char *arg);
+static void opt_log_level_file(pgut_option *opt, const char *arg);
 static void opt_compress_alg(pgut_option *opt, const char *arg);
 
 static pgBackupConfig *cur_config = NULL;
@@ -41,10 +42,10 @@ do_configure(bool show_only)
 	if (replica_timeout != 300)		/* 300 is default value */
 		config->replica_timeout = replica_timeout;
 
-	if (log_to_file != LOGGER_NONE)
-		config->log_to_file = LOG_TO_FILE;
-	if (log_level != LOGGER_NONE)
-		config->log_level = LOG_LEVEL;
+	if (log_level_console != LOG_NONE)
+		config->log_level_console = LOG_LEVEL_CONSOLE;
+	if (log_level_file != LOG_NONE)
+		config->log_level_file = LOG_LEVEL_FILE;
 	if (log_filename)
 		config->log_filename = log_filename;
 	if (error_log_filename)
@@ -90,8 +91,8 @@ pgBackupConfigInit(pgBackupConfig *config)
 	config->master_user = NULL;
 	config->replica_timeout = INT_MIN;	/* INT_MIN means "undefined" */
 
-	config->log_to_file = INT_MIN;		/* INT_MIN means "undefined" */
-	config->log_level = INT_MIN;		/* INT_MIN means "undefined" */
+	config->log_level_console = INT_MIN;	/* INT_MIN means "undefined" */
+	config->log_level_file = INT_MIN;		/* INT_MIN means "undefined" */
 	config->log_filename = NULL;
 	config->error_log_filename = NULL;
 	config->log_directory = NULL;
@@ -108,6 +109,9 @@ pgBackupConfigInit(pgBackupConfig *config)
 void
 writeBackupCatalogConfig(FILE *out, pgBackupConfig *config)
 {
+	uint64		res;
+	const char *unit;
+
 	fprintf(out, "#Backup instance info\n");
 	fprintf(out, "PGDATA = %s\n", config->pgdata);
 	fprintf(out, "system-identifier = %li\n", config->system_identifier);
@@ -131,24 +135,41 @@ writeBackupCatalogConfig(FILE *out, pgBackupConfig *config)
 		fprintf(out, "master-db = %s\n", config->master_db);
 	if (config->master_user)
 		fprintf(out, "master-user = %s\n", config->master_user);
+
 	if (config->replica_timeout != INT_MIN)
-		fprintf(out, "replica_timeout = %d\n", config->replica_timeout);
+	{
+		convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_S,
+								 &res, &unit);
+		fprintf(out, "replica-timeout = " UINT64_FORMAT "%s\n", res, unit);
+	}
 
 	fprintf(out, "#Logging parameters:\n");
-	if (config->log_to_file != INT_MIN)
-		fprintf(out, "log = %d\n", config->log_to_file);
-	if (config->log_level != INT_MIN)
-		fprintf(out, "log-level = %s\n", deparse_log_level(config->log_level));
+	if (config->log_level_console != INT_MIN)
+		fprintf(out, "log-level-console = %s\n", deparse_log_level(config->log_level_console));
+	if (config->log_level_file != INT_MIN)
+		fprintf(out, "log-level-file = %s\n", deparse_log_level(config->log_level_file));
 	if (config->log_filename)
 		fprintf(out, "log-filename = %s\n", config->log_filename);
 	if (config->error_log_filename)
 		fprintf(out, "error-log-filename = %s\n", config->error_log_filename);
 	if (config->log_directory)
 		fprintf(out, "log-directory = %s\n", config->log_directory);
+
+	/*
+	 * Convert values from base unit
+	 */
 	if (config->log_rotation_size)
-		fprintf(out, "log-rotation-size = %d\n", config->log_rotation_size);
+	{
+		convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB,
+								 &res, &unit);
+		fprintf(out, "log-rotation-size = " UINT64_FORMAT "%s\n", res, unit);
+	}
 	if (config->log_rotation_age)
-		fprintf(out, "log-rotation-age = %d\n", config->log_rotation_age);
+	{
+		convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_S,
+								 &res, &unit);
+		fprintf(out, "log-rotation-age = " UINT64_FORMAT "%s\n", res, unit);
+	}
 
 	fprintf(out, "#Retention parameters:\n");
 	if (config->retention_redundancy)
@@ -198,13 +219,13 @@ readBackupCatalogConfigFile(void)
 		{ 'f', 0, "compress-algorithm",		opt_compress_alg,				SOURCE_CMDLINE },
 		{ 'u', 0, "compress-level",			&(config->compress_level),		SOURCE_CMDLINE },
 		/* logging options */
-		{ 'b', 0, "log",					&(config->log_to_file),			SOURCE_CMDLINE },
-		{ 'f', 0, "log-level",				opt_log_level,					SOURCE_CMDLINE },
+		{ 'f', 0, "log-level-console",		opt_log_level_console,			SOURCE_CMDLINE },
+		{ 'f', 0, "log-level-file",			opt_log_level_file,				SOURCE_CMDLINE },
 		{ 's', 0, "log-filename",			&(config->log_filename),		SOURCE_CMDLINE },
 		{ 's', 0, "error-log-filename",		&(config->error_log_filename),	SOURCE_CMDLINE },
 		{ 's', 0, "log-directory",			&(config->log_directory),		SOURCE_CMDLINE },
-		{ 'u', 0, "log-rotation-size",		&(config->log_rotation_size),	SOURCE_CMDLINE },
-		{ 'u', 0, "log-rotation-age",		&(config->log_rotation_age),	SOURCE_CMDLINE },
+		{ 'u', 0, "log-rotation-size",		&(config->log_rotation_size),	SOURCE_CMDLINE,	SOURCE_DEFAULT,	OPTION_UNIT_KB },
+		{ 'u', 0, "log-rotation-age",		&(config->log_rotation_age),	SOURCE_CMDLINE,	SOURCE_DEFAULT,	OPTION_UNIT_S },
 		/* connection options */
 		{ 's', 0, "pgdata",					&(config->pgdata),				SOURCE_FILE_STRICT },
 		{ 's', 0, "pgdatabase",				&(config->pgdatabase),			SOURCE_FILE_STRICT },
@@ -216,7 +237,7 @@ readBackupCatalogConfigFile(void)
 		{ 's', 0, "master-port",			&(config->master_port),			SOURCE_FILE_STRICT },
 		{ 's', 0, "master-db",				&(config->master_db),			SOURCE_FILE_STRICT },
 		{ 's', 0, "master-user",			&(config->master_user),			SOURCE_FILE_STRICT },
-		{ 'u', 0, "replica-timeout",		&(config->replica_timeout),		SOURCE_CMDLINE },
+		{ 'u', 0, "replica-timeout",		&(config->replica_timeout),		SOURCE_CMDLINE,	SOURCE_DEFAULT,	OPTION_UNIT_S },
 		/* other options */
 		{ 'U', 0, "system-identifier",		&(config->system_identifier),	SOURCE_FILE_STRICT },
 		{0}
@@ -234,9 +255,15 @@ readBackupCatalogConfigFile(void)
 }
 
 static void
-opt_log_level(pgut_option *opt, const char *arg)
+opt_log_level_console(pgut_option *opt, const char *arg)
+{
+	cur_config->log_level_console = parse_log_level(arg);
+}
+
+static void
+opt_log_level_file(pgut_option *opt, const char *arg)
 {
-	cur_config->log_level = parse_log_level(arg);
+	cur_config->log_level_file = parse_log_level(arg);
 }
 
 static void

src/configure.c

@@ -140,7 +140,7 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn,
 	/*
 	 * Read the page and verify its header and checksum.
 	 * Under high write load it's possible that we've read partly
-	 * flushed page, so try several times befor throwing an error.
+	 * flushed page, so try several times before throwing an error.
 	 */
 	while(try_checksum--)
 	{
@@ -152,6 +152,20 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn,
 
 		if (read_len != BLCKSZ)
 		{
+			if (read_len == 0)
+			{
+				elog(LOG, "File %s, block %u, file was truncated",
+					 file->path, blknum);
+				return;
+			}
+			else if (try_checksum)
+			{
+				elog(LOG, "File: %s, block %u, expected block size %lu, but read %d, try again",
+					 file->path, blknum, read_len, BLCKSZ);
+				usleep(100);
+				continue;
+			}
+
 			elog(ERROR, "File: %s, invalid block size of block %u : %lu",
 				 file->path, blknum, read_len);
 		}
@@ -295,15 +309,26 @@ backup_data_file(const char *from_root, const char *to_root,
 
 	if ((backup_mode == BACKUP_MODE_DIFF_PAGE ||
 		backup_mode == BACKUP_MODE_DIFF_PTRACK) &&
-		file->pagemap.bitmapsize == 0)
+		file->pagemap.bitmapsize == PageBitmapIsEmpty)
 	{
 		/*
 		 * There are no changed blocks since last backup. We want make
 		 * incremental backup, so we should exit.
 		 */
+		elog(VERBOSE, "Skipping the file because it didn`t changed: %s", file->path);
 		return false;
 	}
 
+	if ((backup_mode == BACKUP_MODE_DIFF_PAGE ||
+		backup_mode == BACKUP_MODE_DIFF_PTRACK) &&
+		file->pagemap.bitmapsize == PageBitmapIsAbsent)
+	{
+		/*
+		 * TODO COMPARE FILE CHECKSUMM to this file version from previous backup
+		 * if they are equal, skip this file
+		 */
+	}
+
 	/* reset size summary */
 	file->read_size = 0;
 	file->write_size = 0;
@@ -357,7 +382,8 @@ backup_data_file(const char *from_root, const char *to_root,
 	 * Read each page, verify checksum and write it to backup.
 	 * If page map is empty backup all pages of the relation.
 	 */
-	if (file->pagemap.bitmapsize == 0)
+	if (file->pagemap.bitmapsize == PageBitmapIsEmpty
+		|| file->pagemap.bitmapsize == PageBitmapIsAbsent)
 	{
 		for (blknum = 0; blknum < nblocks; blknum++)
 		{
@@ -482,7 +508,7 @@ restore_data_file(const char *from_root,
 		if (header.block < blknum)
 			elog(ERROR, "backup is broken at block %u", blknum);
 
-		elog(VERBOSE, "file %s, header compressed size %d", file->path, header.compressed_size);
+		//elog(VERBOSE, "file %s, header compressed size %d", file->path, header.compressed_size);
 		Assert(header.compressed_size <= BLCKSZ);
 
 		read_len = fread(compressed_page.data, 1,

src/data.c

@@ -75,6 +75,8 @@ static char *pgdata_exclude_files[] =
 	"recovery.conf",
 	"postmaster.pid",
 	"postmaster.opts",
+	"backup_label",
+	"tablespace_map",
 	NULL
 };
 
@@ -149,7 +151,7 @@ pgFileInit(const char *path)
 	file->is_datafile = false;
 	file->linked = NULL;
 	file->pagemap.bitmap = NULL;
-	file->pagemap.bitmapsize = 0;
+	file->pagemap.bitmapsize = (current.backup_mode == BACKUP_MODE_DIFF_PAGE) ? PageBitmapIsEmpty : PageBitmapIsAbsent;
 	file->tblspcOid = 0;
 	file->dbOid = 0;
 	file->relOid = 0;
@@ -793,7 +795,7 @@ get_control_value(const char *str, const char *name,
 								 name, str, DATABASE_FILE_LIST);
 
 						*buf_uint64_ptr = '\0';
-						if (!parse_uint64(buf_uint64, value_uint64))
+						if (!parse_uint64(buf_uint64, value_uint64, 0))
 							goto bad_format;
 					}
 

src/dir.c

@@ -66,7 +66,8 @@ help_pg_probackup(void)
 	printf(_("\n  %s init -B backup-path [-l]\n"), PROGRAM_NAME);
 
 	printf(_("\n  %s set-config -B backup-dir --instance=instance_name\n"), PROGRAM_NAME);
-	printf(_("                 [--log-level=log-level]\n"));
+	printf(_("                 [--log-level-console=log-level-console]\n"));
+	printf(_("                 [--log-level-file=log-level-file]\n"));
 	printf(_("                 [--log-filename=log-filename]\n"));
 	printf(_("                 [--error-log-filename=error-log-filename]\n"));
 	printf(_("                 [--log-directory=log-directory]\n"));
@@ -130,9 +131,8 @@ help_pg_probackup(void)
 static void
 help_init(void)
 {
-	printf(_("%s init -B backup-path -D pgdata-dir [-l]\n\n"), PROGRAM_NAME);
+	printf(_("%s init -B backup-path [-l]\n\n"), PROGRAM_NAME);
 	printf(_("  -B, --backup-path=backup-path    location of the backup storage area\n"));
-	printf(_("  -l, --log                        store messages in a log file\n"));
 }
 
 static void
@@ -153,7 +153,6 @@ help_backup(void)
 	printf(_("  -b, --backup-mode=backup-mode    backup mode=FULL|PAGE|PTRACK\n"));
 	printf(_("      --instance=instance_name     name of the instance\n"));
 	printf(_("  -C, --smooth-checkpoint          do smooth checkpoint before backup\n"));
-	printf(_("  -l, --log                        store messages in a log file\n"));
 	printf(_("      --stream                     stream the transaction log and include it in the backup\n"));
 	printf(_("      --archive-timeout            wait timeout for WAL segment archiving\n"));
 	printf(_("  -S, --slot=SLOTNAME              replication slot to use\n"));
@@ -195,7 +194,6 @@ help_restore(void)
 	printf(_("      --instance=instance_name     name of the instance\n"));
 
 	printf(_("  -D, --pgdata=pgdata-dir          location of the database storage area\n"));
-	printf(_("  -l, --log                        store messages in a log file\n"));
 	printf(_("  -i, --backup-id=backup-id        backup to restore\n"));
 
 	printf(_("      --progress                   show progress\n"));
@@ -219,7 +217,6 @@ help_validate(void)
 	printf(_("      --instance=instance_name     name of the instance\n"));
 	printf(_("  -i, --backup-id=backup-id        backup to validate\n"));
 
-	printf(_("  -l, --log                        store messages in a log file\n"));
 	printf(_("      --progress                   show progress\n"));
 	printf(_("      --time=time                  time stamp up to which recovery will proceed\n"));
 	printf(_("      --xid=xid                    transaction ID up to which recovery will proceed\n"));
@@ -250,14 +247,14 @@ help_delete(void)
 	printf(_("  -i, --backup-id=backup-id        backup to delete\n"));
 	printf(_("      --expired                    delete backups expired according to current\n"));
 	printf(_("                                   retention policy\n"));
-	printf(_("  -l, --log                        store messages in a log file\n"));
 }
 
 static void
 help_set_config(void)
 {
 	printf(_("%s set-config -B backup-dir --instance=instance_name\n"), PROGRAM_NAME);
-	printf(_("                 [--log-level=log-level]\n"));
+	printf(_("                 [--log-level-console=log-level-console]\n"));
+	printf(_("                 [--log-level-file=log-level-file]\n"));
 	printf(_("                 [--log-filename=log-filename]\n"));
 	printf(_("                 [--error-log-filename=error-log-filename]\n"));
 	printf(_("                 [--log-directory=log-directory]\n"));
@@ -276,7 +273,10 @@ help_set_config(void)
 	printf(_("      --instance=instance_name     name of the instance\n"));
 
 	printf(_("\n  Logging options:\n"));
-	printf(_("      --log-level=log-level        controls which message levels are sent to the log\n"));
+	printf(_("      --log-level-console=log-level-console\n"
+			 "                                   controls which message levels are sent to the stderr\n"));
+	printf(_("      --log-level-file=log-level-file\n"
+			 "                                   controls which message levels are sent to a log file\n"));
 	printf(_("      --log-filename=log-filename  file names of the created log files which is treated as as strftime pattern\n"));
 	printf(_("      --error-log-filename=error-log-filename\n"));
 	printf(_("                                   file names of the created log files for error messages\n"));