Merge branch 'master' into stable

Author: alubennikova

Makefile

@@ -24,7 +24,9 @@ include $(top_srcdir)/contrib/contrib-global.mk
 endif # USE_PGXS
 
 ifeq ($(top_srcdir),../..)
-srchome=$(top_srcdir)/..
+ ifeq ($(LN_S),ln -s)
+	srchome=$(top_srcdir)/..
+ endif
 else
 srchome=$(top_srcdir)
 endif

README.md

@@ -2,13 +2,13 @@
 
 `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure.
 
-The extension is compatible with:
+The utility is compatible with:
 * PostgreSQL 9.5, 9.6, 10;
 
-`PTRACK` backup requires:
-* Postgres Pro Standard 9.5, 9.6;
-or
-* Postgres Pro Enterprise;
+`PTRACK` backup support provided via following options:
+* vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/3d9ad9ea2568e0f4aaeeda62b59e30f8/raw/8f28e207c1aa172a9597fcda49e69b086c6b42bd/ptrack_9.6.5_v1.3.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/0ce69df847268333b72d72079bd48315/raw/d35d49c28446637ea3fe9dfc1a400bb298bf3318/ptrack_10.0_v1.3.patch)
+* Postgres Pro Standard 9.5, 9.6
+* Postgres Pro Enterprise
 
 As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data:
 * Choosing between full and page-level incremental backups to speed up backup and recovery

src/backup.c

@@ -118,6 +118,7 @@ static XLogRecPtr get_last_ptrack_lsn(void);
 static void check_server_version(void);
 static void check_system_identifiers(void);
 static void confirm_block_size(const char *name, int blcksz);
+static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i);
 
 
 #define disconnect_and_exit(code)				\
@@ -1878,6 +1879,7 @@ backup_files(void *arg)
 		struct stat	buf;
 
 		pgFile *file = (pgFile *) parray_get(arguments->backup_files_list, i);
+		elog(VERBOSE, "Copying file:  \"%s\" ", file->path);
 		if (__sync_lock_test_and_set(&file->lock, 1) != 0)
 			continue;
 
@@ -1918,8 +1920,9 @@ backup_files(void *arg)
 		if (S_ISREG(buf.st_mode))
 		{
 			/* copy the file into backup */
-			if (file->is_datafile)
+			if (file->is_datafile && !file->is_cfs)
 			{
+				/* backup block by block if datafile AND not compressed by cfs*/
 				if (!backup_data_file(arguments->from_root,
 									  arguments->to_root, file,
 									  arguments->prev_backup_start_lsn))
@@ -1961,16 +1964,18 @@ parse_backup_filelist_filenames(parray *files, const char *root)
 	{
 		pgFile	   *file = (pgFile *) parray_get(files, i);
 		char	   *relative;
-		char		*filename = palloc(MAXPGPATH);
+		char		filename[MAXPGPATH];
 		int 		sscanf_result;
 
 		relative = GetRelativePath(file->path, root);
+		filename[0] = '\0';
 
 		elog(VERBOSE, "-----------------------------------------------------: %s", relative);
 		if (path_is_prefix_of_path("global", relative))
 		{
 			file->tblspcOid = GLOBALTABLESPACE_OID;
 			sscanf_result = sscanf(relative, "global/%s", filename);
+			elog(VERBOSE, "global sscanf result: %i", sscanf_result);
 			if (strcmp(relative, "global") == 0)
 			{
 				Assert(S_ISDIR(file->mode));
@@ -1986,6 +1991,7 @@ parse_backup_filelist_filenames(parray *files, const char *root)
 		{
 			file->tblspcOid = DEFAULTTABLESPACE_OID;
 			sscanf_result = sscanf(relative, "base/%u/%s", &(file->dbOid), filename);
+			elog(VERBOSE, "base sscanf result: %i", sscanf_result);
 			if (strcmp(relative, "base") == 0)
 			{
 				Assert(S_ISDIR(file->mode));
@@ -2004,9 +2010,11 @@ parse_backup_filelist_filenames(parray *files, const char *root)
 		}
 		else if (path_is_prefix_of_path(PG_TBLSPC_DIR, relative))
 		{
-			char	*temp_relative_path = palloc(MAXPGPATH);
+			char		temp_relative_path[MAXPGPATH];
 
 			sscanf_result = sscanf(relative, "pg_tblspc/%u/%s", &(file->tblspcOid), temp_relative_path);
+			elog(VERBOSE, "pg_tblspc sscanf result: %i", sscanf_result);
+
 			if (strcmp(relative, "pg_tblspc") == 0)
 			{
 				Assert(S_ISDIR(file->mode));
@@ -2022,21 +2030,43 @@ parse_backup_filelist_filenames(parray *files, const char *root)
 				/*continue parsing */
 				sscanf_result = sscanf(temp_relative_path+strlen(TABLESPACE_VERSION_DIRECTORY)+1, "%u/%s",
 									   &(file->dbOid), filename);
+				elog(VERBOSE, "TABLESPACE_VERSION_DIRECTORY sscanf result: %i", sscanf_result);
 
-				if (sscanf_result == 0)
+				if (sscanf_result == -1)
+				{
+					elog(VERBOSE, "The TABLESPACE_VERSION_DIRECTORY itself, filepath %s", relative);
+				}
+				else if (sscanf_result == 0)
 				{
-					elog(VERBOSE, "the TABLESPACE_VERSION_DIRECTORY itself, filepath %s", relative);
+					/* Found file in pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY
+					   Legal only in case of 'pg_compression'
+					 */
+					if (strcmp(file->name, "pg_compression") == 0)
+					{
+						elog(VERBOSE, "Found pg_compression file in TABLESPACE_VERSION_DIRECTORY, filepath %s", relative);
+						/*Set every datafile in tablespace as is_cfs */
+						set_cfs_datafiles(files, root, relative, i);
+					}
+					else
+					{
+						elog(VERBOSE, "Found illegal file in TABLESPACE_VERSION_DIRECTORY, filepath %s", relative);
+					}
+
 				}
 				else if (sscanf_result == 1)
 				{
 					Assert(S_ISDIR(file->mode));
 					elog(VERBOSE, "dboid %u, filepath %s", file->dbOid, relative);
 					file->is_database = true;
 				}
-				else
+				else if (sscanf_result == 2)
 				{
 					elog(VERBOSE, "dboid %u, filename %s, filepath %s", file->dbOid, filename, relative);
 				}
+				else
+				{
+					elog(VERBOSE, "Illegal file filepath %s", relative);
+				}
 			}
 		}
 		else
@@ -2054,7 +2084,7 @@ parse_backup_filelist_filenames(parray *files, const char *root)
 		}
 
 		/* Check files located inside database directories */
-		if (filename && file->dbOid != 0)
+		if (filename[0] != '\0' && file->dbOid != 0)
 		{
 			if (strcmp(filename, "pg_internal.init") == 0)
 			{
@@ -2081,7 +2111,7 @@ parse_backup_filelist_filenames(parray *files, const char *root)
 				 * Check that and do not mark them with 'is_datafile' flag.
 				 */
 				char *forkNameptr;
-				char *suffix = palloc(MAXPGPATH);;
+				char suffix[MAXPGPATH];
 
 				forkNameptr = strstr(filename, "_");
 				if (forkNameptr != NULL)
@@ -2108,7 +2138,16 @@ parse_backup_filelist_filenames(parray *files, const char *root)
 				{
 					/* first segment of the relfile */
 					elog(VERBOSE, "relOid %u, segno %d, filepath %s", file->relOid, 0, relative);
-					file->is_datafile = true;
+					if (strcmp(relative + strlen(relative) - strlen("cfm"), "cfm") == 0)
+					{
+						/* reloid.cfm */
+						elog(VERBOSE, "Found cfm file %s", relative);
+					}
+					else
+					{
+						elog(VERBOSE, "Found first segment of the relfile %s", relative);
+						file->is_datafile = true;
+					}
 				}
 				else if (sscanf_result == 2)
 				{
@@ -2126,9 +2165,62 @@ parse_backup_filelist_filenames(parray *files, const char *root)
 				}
 			}
 		}
- 	}
+	}
+}
+
+/* If file is equal to pg_compression, then we consider this tablespace as
+ * cfs-compressed and should mark every file in this tablespace as cfs-file
+ * Setting is_cfs is done via going back through 'files' set every file
+ * that contain cfs_tablespace in his path as 'is_cfs'
+ * Goings back through array 'files' is valid option possible because of current
+ * sort rules:
+ * tblspcOid/TABLESPACE_VERSION_DIRECTORY
+ * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid
+ * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1
+ * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1.cfm
+ * tblspcOid/TABLESPACE_VERSION_DIRECTORY/pg_compression
+ */
+static void
+set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i)
+{
+	int len;
+	size_t p;
+	pgFile *prev_file;
+	char *cfs_tblspc_path;
+	char *relative_prev_file;
+
+	cfs_tblspc_path = strdup(relative);
+	len = strlen("/pg_compression");
+	cfs_tblspc_path[strlen(cfs_tblspc_path) - len] = 0;
+	elog(VERBOSE, "CFS DIRECTORY %s, pg_compression path: %s", cfs_tblspc_path, relative);
+
+	for (p = i; p >= 0; p--)
+	{
+		prev_file = (pgFile *) parray_get(files, p);
+		relative_prev_file = GetRelativePath(prev_file->path, root);
+
+		//elog(VERBOSE, "P: %lu, Checking file in cfs tablespace %s", p, relative_prev_file);
+		elog(VERBOSE, "Checking file in cfs tablespace %s", relative_prev_file);
+
+		if (strstr(relative_prev_file, cfs_tblspc_path) != NULL)
+		{
+			if (S_ISREG(prev_file->mode) && prev_file->is_datafile)
+			{
+				elog(VERBOSE, "Setting 'is_cfs' on file %s, name %s",
+					relative_prev_file, prev_file->name);
+				prev_file->is_cfs = true;
+			}
+		}
+		else
+		{
+			elog(VERBOSE, "Breaking on %s", relative_prev_file);
+			break;
+		}
+	}
+	free(cfs_tblspc_path);
 }
 
+
 /*
  * Output the list of files to backup catalog DATABASE_FILE_LIST
  */

src/data.c

@@ -470,6 +470,7 @@ restore_data_file(const char *from_root,
 		if (header.block < blknum)
 			elog(ERROR, "backup is broken at block %u", blknum);
 
+		elog(VERBOSE, "file %s, header compressed size %d", file->path, header.compressed_size);
 		Assert(header.compressed_size <= BLCKSZ);
 
 		read_len = fread(compressed_page.data, 1,

src/dir.c

@@ -393,15 +393,20 @@ dir_list_file_internal(parray *files, const char *root, bool exclude,
 			if (file_name == NULL)
 				file_name = file->path;
 			else
+			{
 				file_name++;
+				file->name = file_name;
+			}
 
 			/* Check if we need to exclude file by name */
 			for (i = 0; pgdata_exclude_files[i]; i++)
-				if (strcmp(file_name, pgdata_exclude_files[i]) == 0)
+				if (strcmp(file->name, pgdata_exclude_files[i]) == 0)
+				{
 					/* Skip */
+					elog(VERBOSE, "Excluding file: %s", file->name);
 					return;
+				}
 		}
-
 		parray_append(files, file);
 	}
 
@@ -463,7 +468,10 @@ dir_list_file_internal(parray *files, const char *root, bool exclude,
 			if (dirname == NULL)
 				dirname = file->path;
 			else
+			{
 				dirname++;
+				file->name = dirname;
+			}
 
 			/*
 			 * If the item in the exclude list starts with '/', compare to the
@@ -472,6 +480,7 @@ dir_list_file_internal(parray *files, const char *root, bool exclude,
 			 */
 			for (i = 0; exclude && pgdata_exclude_dir[i]; i++)
 			{
+				/* Full-path exclude*/
 				if (pgdata_exclude_dir[i][0] == '/')
 				{
 					if (strcmp(file->path, pgdata_exclude_dir[i]) == 0)
@@ -480,14 +489,17 @@ dir_list_file_internal(parray *files, const char *root, bool exclude,
 						break;
 					}
 				}
-				else if (strcmp(dirname, pgdata_exclude_dir[i]) == 0)
+				else if (strcmp(file->name, pgdata_exclude_dir[i]) == 0)
 				{
 					skip = true;
 					break;
 				}
 			}
 			if (skip)
+			{
+				elog(VERBOSE, "Excluding directory content: %s", file->name);
 				break;
+			}
 		}
 
 		/* open directory and list contents */
@@ -678,9 +690,11 @@ print_file_list(FILE *out, const parray *files, const char *root)
 			path = GetRelativePath(path, root);
 
 		fprintf(out, "{\"path\":\"%s\", \"size\":\"%lu\",\"mode\":\"%u\","
-					 "\"is_datafile\":\"%u\", \"crc\":\"%u\", \"compress_alg\":\"%s\"",
+					 "\"is_datafile\":\"%u\", \"is_cfs\":\"%u\", \"crc\":\"%u\","
+					 "\"compress_alg\":\"%s\"",
 				path, (unsigned long) file->write_size, file->mode,
-				file->is_datafile?1:0, file->crc, deparse_compress_alg(file->compress_alg));
+				file->is_datafile?1:0, file->is_cfs?1:0, file->crc,
+				deparse_compress_alg(file->compress_alg));
 
 		if (file->is_datafile)
 			fprintf(out, ",\"segno\":\"%d\"", file->segno);
@@ -853,6 +867,7 @@ dir_read_file_list(const char *root, const char *file_txt)
 		uint64		write_size,
 					mode,		/* bit length of mode_t depends on platforms */
 					is_datafile,
+					is_cfs,
 					crc,
 					segno;
 		pgFile	   *file;
@@ -861,6 +876,7 @@ dir_read_file_list(const char *root, const char *file_txt)
 		get_control_value(buf, "size", NULL, &write_size, true);
 		get_control_value(buf, "mode", NULL, &mode, true);
 		get_control_value(buf, "is_datafile", NULL, &is_datafile, true);
+		get_control_value(buf, "is_cfs", NULL, &is_cfs, false);
 		get_control_value(buf, "crc", NULL, &crc, true);
 
 		/* optional fields */
@@ -878,6 +894,7 @@ dir_read_file_list(const char *root, const char *file_txt)
 		file->write_size = (size_t) write_size;
 		file->mode = (mode_t) mode;
 		file->is_datafile = is_datafile ? true : false;
+		file->is_cfs = is_cfs ? true : false;
 		file->crc = (pg_crc32) crc;
 		file->compress_alg = parse_compress_alg(compress_alg_string);
 		if (linked[0])

src/pg_probackup.c

@@ -17,7 +17,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
-const char *PROGRAM_VERSION	= "2.0.8";
+const char *PROGRAM_VERSION	= "2.0.10";
 const char *PROGRAM_URL		= "https://github.com/postgrespro/pg_probackup";
 const char *PROGRAM_EMAIL	= "https://github.com/postgrespro/pg_probackup/issues";
 

src/pg_probackup.h

@@ -80,6 +80,7 @@ typedef enum CompressAlg
 /* Information about single file (or dir) in backup */
 typedef struct pgFile
 {
+	char	*name;			/* file or directory name */
 	mode_t	mode;			/* protection (file type and permission) */
 	size_t	size;			/* size of the file */
 	size_t	read_size;		/* size of the portion read (if only some pages are
@@ -90,7 +91,7 @@ typedef struct pgFile
 	pg_crc32 crc;			/* CRC value of the file, regular file only */
 	char	*linked;		/* path of the linked file */
 	bool	is_datafile;	/* true if the file is PostgreSQL data file */
-	char	*path;			/* path of the file */
+	char	*path;			/* absolute path of the file */
 	Oid		tblspcOid;		/* tblspcOid extracted from path, if applicable */
 	Oid		dbOid;			/* dbOid extracted from path, if applicable */
 	Oid		relOid;			/* relOid extracted from path, if applicable */

src/restore.c

@@ -686,7 +686,8 @@ restore_files(void *arg)
 		 * block and have BackupPageHeader meta information, so we cannot just
 		 * copy the file from backup.
 		 */
-		if (file->is_datafile)
+		elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", file->path, file->is_datafile?1:0, file->is_cfs?1:0);
+		if (file->is_datafile && !file->is_cfs)
 			restore_data_file(from_root, pgdata, file, arguments->backup);
 		else
 			copy_file(from_root, pgdata, file);

src/utils/logger.c

@@ -86,6 +86,9 @@ write_elevel(FILE *stream, int elevel)
 {
 	switch (elevel)
 	{
+		case VERBOSE:
+			fputs("VERBOSE: ", stream);
+			break;
 		case LOG:
 			fputs("LOG: ", stream);
 			break;