PGPRO-1457: backup mode DELTA

Author: gsmolk

src/backup.c

@@ -479,7 +479,8 @@ do_backup_instance(void)
 	 * backup on current timeline exists and get its filelist.
 	 */
 	if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
-		current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
+		current.backup_mode == BACKUP_MODE_DIFF_PTRACK ||
+		current.backup_mode == BACKUP_MODE_DIFF_DELTA)
 	{
 		parray	   *backup_list;
 		/* get list of backups already taken */

src/catalog.c

@@ -21,7 +21,7 @@
 #include <time.h>
 #include <unistd.h>
 
-static const char *backupModes[] = {"", "PAGE", "PTRACK", "FULL"};
+static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"};
 static pgBackup *readBackupControlFile(const char *path);
 
 static bool exit_hook_registered = false;
@@ -593,6 +593,8 @@ parse_backup_mode(const char *value)
 		return BACKUP_MODE_DIFF_PAGE;
 	else if (len > 0 && pg_strncasecmp("ptrack", v, len) == 0)
 		return BACKUP_MODE_DIFF_PTRACK;
+	else if (len > 0 && pg_strncasecmp("delta", v, len) == 0)
+		return BACKUP_MODE_DIFF_DELTA;
 
 	/* Backup mode is invalid, so leave with an error */
 	elog(ERROR, "invalid backup-mode \"%s\"", value);
@@ -610,6 +612,8 @@ deparse_backup_mode(BackupMode mode)
 			return "page";
 		case BACKUP_MODE_DIFF_PTRACK:
 			return "ptrack";
+		case BACKUP_MODE_DIFF_DELTA:
+			return "delta";
 		case BACKUP_MODE_INVALID:
 			return "invalid";
 	}

src/data.c

@@ -132,11 +132,10 @@ parse_page(Page page, XLogRecPtr *lsn)
  */
 static int
 read_page_from_file(pgFile *file, BlockNumber blknum,
-					FILE *in, Page page)
+					FILE *in, Page page, XLogRecPtr *page_lsn)
 {
 	off_t				offset = blknum*BLCKSZ;
 	size_t				read_len = 0;
-	XLogRecPtr			page_lsn;
 
 	/* read the block */
 	if (fseek(in, offset, SEEK_SET) != 0)
@@ -167,7 +166,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum,
 	 * If after several attempts page header is still invalid, throw an error.
 	 * The same idea is applied to checksum verification.
 	 */
-	if (!parse_page(page, &page_lsn))
+	if (!parse_page(page, page_lsn))
 	{
 		int i;
 		/* Check if the page is zeroed. */
@@ -232,6 +231,7 @@ backup_data_page(backup_files_args *arguments,
 	BackupPageHeader	header;
 	Page 				page = malloc(BLCKSZ);
 	Page 				compressed_page = NULL;
+	XLogRecPtr			page_lsn = 0;
 	size_t				write_buffer_size;
 	char				write_buffer[BLCKSZ+sizeof(header)];
 
@@ -252,7 +252,7 @@ backup_data_page(backup_files_args *arguments,
 		while(!page_is_valid && try_again)
 		{
 			int result = read_page_from_file(file, blknum,
-											 in, page);
+											 in, page, &page_lsn);
 
 			try_again--;
 			if (result == 0)
@@ -316,6 +316,20 @@ backup_data_page(backup_files_args *arguments,
 			if (is_checksum_enabled)
 				((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum);
 		}
+		/* get lsn from page */
+		if (!parse_page(page, &page_lsn))
+			elog(ERROR, "Cannot parse page after pg_ptrack_get_block. "
+							"Possible risk of a memory corruption");
+
+	}
+
+	if (backup_mode == BACKUP_MODE_DIFF_DELTA &&
+				header.compressed_size != PageIsTruncated &&
+								page_lsn < prev_backup_start_lsn)
+	{
+		elog(VERBOSE, "Skipping blknum: %u in file: %s", blknum, file->path);
+		free(page);
+		return;
 	}
 
 	if (header.compressed_size != PageIsTruncated)
@@ -478,6 +492,8 @@ backup_data_file(backup_files_args* arguments,
 							 &n_blocks_skipped, backup_mode);
 			n_blocks_read++;
 		}
+		if (backup_mode == BACKUP_MODE_DIFF_DELTA)
+			file->n_blocks = n_blocks_read;
 	}
 	/* If page map is not empty we scan only changed blocks, */
 	else
@@ -545,6 +561,7 @@ restore_data_file(const char *from_root,
 	FILE			   *out;
 	BackupPageHeader	header;
 	BlockNumber			blknum;
+	size_t              file_size;
 
 	/* open backup mode file for read */
 	in = fopen(file->path, "r");
@@ -652,6 +669,31 @@ restore_data_file(const char *from_root,
 		}
 	}
 
+    /*
+     * DELTA backup has no knowledge about truncated blocks as PAGE or PTRACK do
+     * but knows file size at the time of backup.
+     * So when restoring file from delta backup we, knowning it`s size at
+     * a time of a backup, can truncate file to this size.
+     */
+
+	if (backup->backup_mode == BACKUP_MODE_DIFF_DELTA)
+	{
+		/* get file current size */
+		fseek(out, 0, SEEK_END);
+		file_size = ftell(out);
+
+		if (file_size > file->n_blocks * BLCKSZ)
+		{
+			/*
+			 * Truncate file to this length.
+			 */
+			if (ftruncate(fileno(out), file->n_blocks * BLCKSZ) != 0)
+				elog(ERROR, "cannot truncate \"%s\": %s",
+					 file->path, strerror(errno));
+			elog(INFO, "Delta truncate file %s to block %u", file->path, file->n_blocks);
+		}
+	}
+
 	/* update file permission */
 	if (chmod(to_path, file->mode) == -1)
 	{

src/dir.c

@@ -166,6 +166,7 @@ pgFileInit(const char *path)
 	file->path = pgut_malloc(strlen(path) + 1);
 	strcpy(file->path, path);		/* enough buffer size guaranteed */
 	file->is_cfs = false;
+	file->n_blocks = -1;			/* can change only in DELTA backups */
 	file->compress_alg = NOT_DEFINED_COMPRESS;
 	return file;
 }
@@ -721,6 +722,9 @@ print_file_list(FILE *out, const parray *files, const char *root)
 		if (S_ISLNK(file->mode))
 			fprintf(out, ",\"linked\":\"%s\"", file->linked);
 
+		if (file->n_blocks != -1)
+			fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks);
+
 		fprintf(out, "}\n");
 	}
 }
@@ -888,7 +892,8 @@ dir_read_file_list(const char *root, const char *file_txt)
 					is_datafile,
 					is_cfs,
 					crc,
-					segno;
+					segno,
+					n_blocks;
 		pgFile	   *file;
 
 		get_control_value(buf, "path", path, NULL, true);
@@ -902,6 +907,7 @@ dir_read_file_list(const char *root, const char *file_txt)
 		get_control_value(buf, "linked", linked, NULL, false);
 		get_control_value(buf, "segno", NULL, &segno, false);
 		get_control_value(buf, "compress_alg", compress_alg_string, NULL, false);
+		get_control_value(buf, "n_blocks", NULL, &n_blocks, false);
 
 		if (root)
 			join_path_components(filepath, root, path);
@@ -919,6 +925,7 @@ dir_read_file_list(const char *root, const char *file_txt)
 		if (linked[0])
 			file->linked = pgut_strdup(linked);
 		file->segno = (int) segno;
+		file->n_blocks = (int) n_blocks;
 
 		parray_append(files, file);
 	}

src/help.c

@@ -190,7 +190,7 @@ help_backup(void)
 	printf(_("                 [--replica-timeout=timeout]\n\n"));
 
 	printf(_("  -B, --backup-path=backup-path    location of the backup storage area\n"));
-	printf(_("  -b, --backup-mode=backup-mode    backup mode=FULL|PAGE|PTRACK\n"));
+	printf(_("  -b, --backup-mode=backup-mode    backup mode=FULL|PAGE|DELTA|PTRACK\n"));
 	printf(_("      --instance=instance_name     name of the instance\n"));
 	printf(_("  -C, --smooth-checkpoint          do smooth checkpoint before backup\n"));
 	printf(_("      --stream                     stream the transaction log and include it in the backup\n"));

src/pg_probackup.h

@@ -83,6 +83,7 @@ typedef struct pgFile
 	char	*name;			/* file or directory name */
 	mode_t	mode;			/* protection (file type and permission) */
 	size_t	size;			/* size of the file */
+	int	n_blocks;			/* size of the file in blocks, readed during DELTA backup */
 	size_t	read_size;		/* size of the portion read (if only some pages are
 							   backed up, it's different from size) */
 	size_t	write_size;		/* size of the backed-up file. BYTES_INVALID means
@@ -126,7 +127,8 @@ typedef enum BackupMode
 {
 	BACKUP_MODE_INVALID = 0,
 	BACKUP_MODE_DIFF_PAGE,		/* incremental page backup */
-	BACKUP_MODE_DIFF_PTRACK,	/* incremental page backup with ptrack system*/
+	BACKUP_MODE_DIFF_PTRACK,	/* incremental page backup with ptrack system */
+	BACKUP_MODE_DIFF_DELTA,		/* incremental page backup with lsn comparison */
 	BACKUP_MODE_FULL			/* full backup */
 } BackupMode;
 

src/validate.c

@@ -51,7 +51,8 @@ pgBackupValidate(pgBackup *backup)
 
 	if (backup->backup_mode != BACKUP_MODE_FULL &&
 		backup->backup_mode != BACKUP_MODE_DIFF_PAGE &&
-		backup->backup_mode != BACKUP_MODE_DIFF_PTRACK)
+		backup->backup_mode != BACKUP_MODE_DIFF_PTRACK &&
+		backup->backup_mode != BACKUP_MODE_DIFF_DELTA)
 		elog(INFO, "Invalid backup_mode of backup %s", base36enc(backup->start_time));
 
 	pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR);