PGPRO-1457: fix DELTA backup not copiying files created by CREATE DATABASE

Author: gsmolk

src/backup.c

@@ -496,7 +496,8 @@ do_backup_instance(void)
 
 		pgBackupGetPath(prev_backup, prev_backup_filelist_path, lengthof(prev_backup_filelist_path),
 						DATABASE_FILE_LIST);
-		prev_backup_filelist = dir_read_file_list(pgdata, prev_backup_filelist_path);
+		/* Files of previous backup needed by DELTA backup */
+		prev_backup_filelist = dir_read_file_list(NULL, prev_backup_filelist_path);
 
 		/* If lsn is not NULL, only pages with higher lsn will be copied. */
 		prev_backup_start_lsn = prev_backup->start_lsn;
@@ -1946,6 +1947,25 @@ backup_files(void *arg)
 
 		if (S_ISREG(buf.st_mode))
 		{
+			/* Check that file exist in previous backup */
+			if (current.backup_mode == BACKUP_MODE_DIFF_DELTA)
+			{
+				int			p;
+				char	   *relative;
+				int n_prev_backup_files_list = parray_num(arguments->prev_backup_filelist);
+				relative = GetRelativePath(file->path, arguments->from_root);
+				for (p = 0; p < n_prev_backup_files_list; p++)
+				{
+					pgFile *prev_file = (pgFile *) parray_get(arguments->prev_backup_filelist, p);
+					if (strcmp(relative, prev_file->path) == 0)
+					{
+						/* File exists in previous backup */
+						file->exists_in_prev = true;
+						elog(INFO, "File exists at the time of previous backup %s", relative);
+						break;
+					}
+				}
+			}
 			/* copy the file into backup */
 			if (file->is_datafile && !file->is_cfs)
 			{

src/data.c

@@ -317,17 +317,22 @@ backup_data_page(backup_files_args *arguments,
 				((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum);
 		}
 		/* get lsn from page */
-		if (!parse_page(page, &page_lsn))
+		if (backup_mode == BACKUP_MODE_DIFF_DELTA &&
+				file->exists_in_prev &&
+					header.compressed_size != PageIsTruncated &&
+						!parse_page(page, &page_lsn))
 			elog(ERROR, "Cannot parse page after pg_ptrack_get_block. "
 							"Possible risk of a memory corruption");
 
 	}
 
 	if (backup_mode == BACKUP_MODE_DIFF_DELTA &&
+			file->exists_in_prev &&
 				header.compressed_size != PageIsTruncated &&
-								page_lsn < prev_backup_start_lsn)
+						page_lsn < prev_backup_start_lsn)
 	{
 		elog(VERBOSE, "Skipping blknum: %u in file: %s", blknum, file->path);
+		(*n_skipped)++;
 		free(page);
 		return;
 	}
@@ -557,18 +562,22 @@ restore_data_file(const char *from_root,
 				  pgBackup *backup)
 {
 	char				to_path[MAXPGPATH];
-	FILE			   *in;
-	FILE			   *out;
+	FILE			   *in = NULL;
+	FILE			   *out = NULL;
 	BackupPageHeader	header;
 	BlockNumber			blknum;
 	size_t              file_size;
 
-	/* open backup mode file for read */
-	in = fopen(file->path, "r");
-	if (in == NULL)
+	/* BYTES_INVALID allowed only in case of restoring file from DELTA backup */
+	if (file->write_size != BYTES_INVALID)
 	{
-		elog(ERROR, "cannot open backup file \"%s\": %s", file->path,
-			 strerror(errno));
+		/* open backup mode file for read */
+		in = fopen(file->path, "r");
+		if (in == NULL)
+		{
+			elog(ERROR, "cannot open backup file \"%s\": %s", file->path,
+				 strerror(errno));
+		}
 	}
 
 	/*
@@ -594,6 +603,10 @@ restore_data_file(const char *from_root,
 		DataPage	compressed_page; /* used as read buffer */
 		DataPage	page;
 
+		/* File didn`t changed. Nothig to copy */
+		if (file->write_size == BYTES_INVALID)
+			break;
+
 		/* read BackupPageHeader */
 		read_len = fread(&header, 1, sizeof(header), in);
 		if (read_len != sizeof(header))
@@ -670,9 +683,10 @@ restore_data_file(const char *from_root,
 	}
 
     /*
-     * DELTA backup has no knowledge about truncated blocks as PAGE or PTRACK do
-     * but knows file size at the time of backup.
-     * So when restoring file from delta backup we, knowning it`s size at
+     * DELTA backup have no knowledge about truncated blocks as PAGE or PTRACK do
+     * But during DELTA backup we read every file in PGDATA and thus DELTA backup
+     * knows exact size of every file at the time of backup.
+     * So when restoring file from DELTA backup we, knowning it`s size at
      * a time of a backup, can truncate file to this size.
      */
 
@@ -681,7 +695,6 @@ restore_data_file(const char *from_root,
 		/* get file current size */
 		fseek(out, 0, SEEK_END);
 		file_size = ftell(out);
-
 		if (file_size > file->n_blocks * BLCKSZ)
 		{
 			/*
@@ -699,7 +712,8 @@ restore_data_file(const char *from_root,
 	{
 		int errno_tmp = errno;
 
-		fclose(in);
+		if (in)
+			fclose(in);
 		fclose(out);
 		elog(ERROR, "cannot change mode of \"%s\": %s", to_path,
 			 strerror(errno_tmp));
@@ -709,7 +723,8 @@ restore_data_file(const char *from_root,
 		fsync(fileno(out)) != 0 ||
 		fclose(out))
 		elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno));
-	fclose(in);
+	if (in)
+		fclose(in);
 }
 
 /*

src/dir.c

@@ -166,7 +166,8 @@ pgFileInit(const char *path)
 	file->path = pgut_malloc(strlen(path) + 1);
 	strcpy(file->path, path);		/* enough buffer size guaranteed */
 	file->is_cfs = false;
-	file->n_blocks = -1;			/* can change only in DELTA backups */
+	file->exists_in_prev = false;	/* can change only in Incremental backup. */
+	file->n_blocks = -1;			/* can change only in DELTA backup. Number of blocks readed during backup */
 	file->compress_alg = NOT_DEFINED_COMPRESS;
 	return file;
 }

src/pg_probackup.h

@@ -83,7 +83,6 @@ typedef struct pgFile
 	char	*name;			/* file or directory name */
 	mode_t	mode;			/* protection (file type and permission) */
 	size_t	size;			/* size of the file */
-	int	n_blocks;			/* size of the file in blocks, readed during DELTA backup */
 	size_t	read_size;		/* size of the portion read (if only some pages are
 							   backed up, it's different from size) */
 	size_t	write_size;		/* size of the backed-up file. BYTES_INVALID means
@@ -98,8 +97,10 @@ typedef struct pgFile
 	Oid		relOid;			/* relOid extracted from path, if applicable */
 	char	*forkName;		/* forkName extracted from path, if applicable */
 	int		segno;			/* Segment number for ptrack */
+	int		n_blocks;		/* size of the file in blocks, readed during DELTA backup */
 	bool	is_cfs;			/* Flag to distinguish files compressed by CFS*/
 	bool	is_database;
+	bool	exists_in_prev;	/* Mark files, both data and regular, that exists in previous backup */
 	CompressAlg compress_alg; /* compression algorithm applied to the file */
 	volatile uint32 lock;	/* lock for synchronization of parallel threads  */
 	datapagemap_t pagemap;	/* bitmap of pages updated since previous backup */

src/restore.c

@@ -378,16 +378,25 @@ restore_backup(pgBackup *backup)
 	pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR);
 	pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST);
 	files = dir_read_file_list(database_path, list_path);
-	for (i = parray_num(files) - 1; i >= 0; i--)
-	{
-		pgFile *file = (pgFile *) parray_get(files, i);
 
-		/*
-		 * Remove files which haven't changed since previous backup
-		 * and was not backed up
-		 */
-		if (file->write_size == BYTES_INVALID)
-			pgFileFree(parray_remove(files, i));
+	/*
+	 * For PAGE and PTRACK backups remove files which haven't changed
+	 * since previous backup and thus were not backed up.
+	 * We can`t do the same when restoring DELTA backup because we need information
+	 * about every file to correctly truncate them.
+	 */
+	if (backup->backup_mode == BACKUP_MODE_DIFF_PAGE ||
+			backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)
+	{
+		for (i = parray_num(files) - 1; i >= 0; i--)
+		{
+			pgFile *file = (pgFile *) parray_get(files, i);
+			if (file->write_size == BYTES_INVALID)
+			{
+				elog(VERBOSE, "The file didn`t changed. Skip restore: %s", file->path);
+				pgFileFree(parray_remove(files, i));
+			}
+		}
 	}
 
 	/* setup threads */
@@ -719,21 +728,14 @@ restore_files(void *arg)
 		/* Directories was created before */
 		if (S_ISDIR(file->mode))
 		{
-			elog(LOG, "directory, skip");
-			continue;
-		}
-
-		/* not backed up */
-		if (file->write_size == BYTES_INVALID)
-		{
-			elog(LOG, "not backed up, skip");
+			elog(VERBOSE, "directory, skip");
 			continue;
 		}
 
 		/* Do not restore tablespace_map file */
 		if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, rel_path))
 		{
-			elog(LOG, "skip tablespace_map");
+			elog(VERBOSE, "skip tablespace_map");
 			continue;
 		}
 
@@ -750,8 +752,9 @@ restore_files(void *arg)
 			copy_file(from_root, pgdata, file);
 
 		/* print size of restored file */
-		elog(LOG, "Restored file %s : %lu bytes",
-			 file->path, (unsigned long) file->write_size);
+		if (file->write_size != BYTES_INVALID)
+			elog(LOG, "Restored file %s : %lu bytes",
+				file->path, (unsigned long) file->write_size);
 	}
 }