[Issue #60] Calculate compression ratio("zratio") during backup

Author: gsmolk

Documentation.md

@@ -708,13 +708,14 @@ pg_probackup displays the list of all the available backups. For example:
 
 ```
 BACKUP INSTANCE 'node'
-============================================================================================================================================
- Instance    Version  ID      Recovery time           Mode    WAL      Current/Parent TLI    Time    Data   Start LSN    Stop LSN    Status 
-============================================================================================================================================
- node        10       P7XDQV  2018-04-29 05:32:59+03  DELTA   STREAM     1 / 1                11s    19MB   0/15000060   0/15000198  OK
- node        10       P7XDJA  2018-04-29 05:28:36+03  PTRACK  STREAM     1 / 1                21s    32MB   0/13000028   0/13000198  OK
- node        10       P7XDHU  2018-04-29 05:27:59+03  PAGE    STREAM     1 / 1                31s    33MB   0/11000028   0/110001D0  OK
- node        10       P7XDHB  2018-04-29 05:27:15+03  FULL    STREAM     1 / 0                11s    39MB   0/F000028    0/F000198   OK
+======================================================================================================================================
+ Instance  Version  ID      Recovery time           Mode    WAL Mode  TLI  Time    Data   WAL  Zratio  Start LSN   Stop LSN    Status
+======================================================================================================================================
+ node      10       PYSUE8  2019-10-03 15:51:48+03  FULL    ARCHIVE   1/0   16s  9047kB  16MB    4.31  0/12000028  0/12000160  OK
+ node      10       P7XDQV  2018-04-29 05:32:59+03  DELTA   STREAM    1/1   11s    19MB  16MB    1.00  0/15000060  0/15000198  OK
+ node      10       P7XDJA  2018-04-29 05:28:36+03  PTRACK  STREAM    1/1   21s    32MB  32MB    1.00  0/13000028  0/13000198  OK
+ node      10       P7XDHU  2018-04-29 05:27:59+03  PAGE    STREAM    1/1   15s    33MB  16MB    1.00  0/11000028  0/110001D0  OK
+ node      10       P7XDHB  2018-04-29 05:27:15+03  FULL    STREAM    1/0   11s    39MB  16MB    1.00  0/F000028   0/F000198   OK
 ```
 
 For each backup, the following information is provided:
@@ -724,12 +725,14 @@ For each backup, the following information is provided:
 - ID — the backup identifier.
 - Recovery time — the earliest moment for which you can restore the state of the database cluster.
 - Mode — the method used to take this backup. Possible values: FULL, PAGE, DELTA, PTRACK.
-- WAL — the WAL delivery mode. Possible values: STREAM and ARCHIVE.
-- Current/Parent TLI — timeline identifiers of current backup and its parent.
+- WAL Mode — the WAL delivery mode. Possible values: STREAM and ARCHIVE.
+- TLI — timeline identifiers of current backup and its parent.
 - Time — the time it took to perform the backup.
-- Data — the size of the data files in this backup. This value does not include the size of WAL files.
-- Start LSN — WAL log sequence number corresponding to the start of the backup process.
-- Stop LSN — WAL log sequence number corresponding to the end of the backup process.
+- Data — the size of the data files in this backup. This value does not include the size of WAL files. In case of STREAM backup the total size of backup can be calculated as 'Data' + 'WAL'.
+- WAL — the uncompressed size of WAL files required to apply by PostgreSQL recovery process to reach consistency.
+- Zratio — compression ratio calculated as 'uncompressed-bytes' / 'data-bytes'.
+- Start LSN — WAL log sequence number corresponding to the start of the backup process. REDO point for PostgreSQL recovery process to start from.
+- Stop LSN — WAL log sequence number corresponding to the end of the backup process. Consistency point for PostgreSQL recovery process.
 - Status — backup status. Possible values:
 
     - OK — the backup is complete and valid.
@@ -774,11 +777,29 @@ recovery-xid = 597
 recovery-time = '2017-05-16 12:57:31'
 data-bytes = 22288792
 wal-bytes = 16777216
+uncompressed-bytes = 39961833
+pgdata-bytes = 39859393
 status = OK
 parent-backup-id = 'PT8XFX'
 primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any'
 ```
 
+Detailed output has additional attributes:
+- compress-alg — compression algorithm used during backup. Possible values: 'zlib', 'pglz', 'none'.
+- compress-level — compression level used during backup.
+- from-replica — the fact that backup was taken from standby server. Possible values: '1', '0'.
+- block-size — (block_size)[https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE] setting of PostgreSQL cluster at the moment of backup start.
+- wal-block-size — (wal_block_size)[https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE] setting of PostgreSQL cluster at the moment of backup start.
+- checksum-version — the fact that PostgreSQL cluster, from which backup is taken, has enabled [data block checksumms](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS). Possible values: '1', '0'.
+- program-version — full version of pg_probackup binary used to create backup.
+- start-time — the backup starting time.
+- end-time — the backup ending time.
+- uncompressed-bytes — size of the data files before adding page headers and applying compression. You can evaluate the effectiveness of compression by comparing 'uncompressed-bytes' to 'data-bytes' if compression if used.
+- pgdata-bytes — size of the PostgreSQL cluster data files at the time of backup. You can evaluate the effectiveness of incremental backup by comparing 'pgdata-bytes' to 'uncompressed-bytes'.
+- recovery-xid — current transaction id at the moment of backup ending.
+- parent-backup-id — backup ID of parent backup. Available only for incremental backups.
+- primary_conninfo — libpq conninfo used for connection to PostgreSQL cluster during backup. The password is not included.
+
 To get more detailed information about the backup in json format, run the show with the backup ID:
 
     pg_probackup show -B backup_dir --instance instance_name --format=json -i backup_id
@@ -851,7 +872,7 @@ For each backup, the following information is provided:
 - Max Segno — number of the last existing WAL segment belonging to the timeline.
 - N segments — number of WAL segments belonging to the timeline.
 - Size — the size files take on disk.
-- Zratio - compression ratio calculated as "N segments" * wal_seg_size / "Size".
+- Zratio — compression ratio calculated as 'N segments' * wal_seg_size / 'Size'.
 - N backups — number of backups belonging to the timeline. To get the details about backups, use json format.
 - Status — archive status for this exact timeline. Possible values:
 	- OK — all WAL segments between Min and Max are present.

src/backup.c

@@ -547,6 +547,22 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo)
 	/* close ssh session in main thread */
 	fio_disconnect();
 
+	/* Calculate pgdata_bytes */
+	for (i = 0; i < parray_num(backup_files_list); i++)
+	{
+		pgFile	   *file = (pgFile *) parray_get(backup_files_list, i);
+
+		/* In case of FULL or DELTA backup we can trust read_size.
+		 * In case of PAGE or PTRACK we are forced to trust datafile size,
+		 * taken at the start of backup.
+		 */
+		if (current.backup_mode == BACKUP_MODE_FULL ||
+			current.backup_mode == BACKUP_MODE_DIFF_DELTA)
+			current.pgdata_bytes += file->read_size;
+		else
+			current.pgdata_bytes += file->size;
+	}
+
 	/* Add archived xlog files into the list of files of this backup */
 	if (stream_wal)
 	{
@@ -1934,6 +1950,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
 				file->crc = pgFileGetCRC(file->path, true, false,
 										 &file->read_size, FIO_BACKUP_HOST);
 				file->write_size = file->read_size;
+				file->uncompressed_size = file->read_size;
 				free(file->path);
 				file->path = strdup(PG_BACKUP_LABEL_FILE);
 				parray_append(backup_files_list, file);

src/catalog.c

@@ -1052,6 +1052,12 @@ pgBackupWriteControl(FILE *out, pgBackup *backup)
 	if (backup->wal_bytes != BYTES_INVALID)
 		fio_fprintf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes);
 
+	if (backup->uncompressed_bytes >= 0)
+		fio_fprintf(out, "uncompressed-bytes = " INT64_FORMAT "\n", backup->uncompressed_bytes);
+
+	if (backup->pgdata_bytes >= 0)
+		fio_fprintf(out, "pgdata-bytes = " INT64_FORMAT "\n", backup->pgdata_bytes);
+
 	fio_fprintf(out, "status = %s\n", status2str(backup->status));
 
 	/* 'parent_backup' is set if it is incremental backup */
@@ -1121,6 +1127,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 	char		buf[BUFFERSZ];
 	size_t		write_len = 0;
 	int64 		backup_size_on_disk = 0;
+	int64 		uncompressed_size_on_disk = 0;
 	int64 		wal_size_on_disk = 0;
 
 	pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST);
@@ -1142,16 +1149,25 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 		i++;
 
 		if (S_ISDIR(file->mode))
+		{
 			backup_size_on_disk += 4096;
+			uncompressed_size_on_disk += 4096;
+		}
 
 		/* Count the amount of the data actually copied */
 		if (S_ISREG(file->mode) && file->write_size > 0)
 		{
-			/* TODO: in 3.0 add attribute is_walfile */
+			/*
+			 * Size of WAL files in 'pg_wal' is counted separately
+			 * TODO: in 3.0 add attribute is_walfile
+			 */
 			if (IsXLogFileName(file->name) && (file->external_dir_num == 0))
 				wal_size_on_disk += file->write_size;
 			else
+			{
 				backup_size_on_disk += file->write_size;
+				uncompressed_size_on_disk += file->uncompressed_size;
+			}
 		}
 
 		/* for files from PGDATA and external files use rel_path
@@ -1235,6 +1251,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 	/* use extra variable to avoid reset of previous data_bytes value in case of error */
 	backup->data_bytes = backup_size_on_disk;
 	backup->wal_bytes = wal_size_on_disk;
+	backup->uncompressed_bytes = uncompressed_size_on_disk;
 }
 
 /*
@@ -1269,6 +1286,8 @@ readBackupControlFile(const char *path)
 		{'t', 0, "recovery-time",		&backup->recovery_time, SOURCE_FILE_STRICT},
 		{'I', 0, "data-bytes",			&backup->data_bytes, SOURCE_FILE_STRICT},
 		{'I', 0, "wal-bytes",			&backup->wal_bytes, SOURCE_FILE_STRICT},
+		{'I', 0, "uncompressed-bytes",	&backup->uncompressed_bytes, SOURCE_FILE_STRICT},
+		{'I', 0, "pgdata-bytes",		&backup->pgdata_bytes, SOURCE_FILE_STRICT},
 		{'u', 0, "block-size",			&backup->block_size, SOURCE_FILE_STRICT},
 		{'u', 0, "xlog-block-size",		&backup->wal_block_size, SOURCE_FILE_STRICT},
 		{'u', 0, "checksum-version",	&backup->checksum_version, SOURCE_FILE_STRICT},
@@ -1513,6 +1532,8 @@ pgBackupInit(pgBackup *backup)
 
 	backup->data_bytes = BYTES_INVALID;
 	backup->wal_bytes = BYTES_INVALID;
+	backup->uncompressed_bytes = 0;
+	backup->pgdata_bytes = 0;
 
 	backup->compress_alg = COMPRESS_ALG_DEFAULT;
 	backup->compress_level = COMPRESS_LEVEL_DEFAULT;

src/data.c

@@ -510,6 +510,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum,
 	}
 
 	file->write_size += write_buffer_size;
+	file->uncompressed_size += BLCKSZ;
 }
 
 /*
@@ -556,6 +557,7 @@ backup_data_file(backup_files_arg* arguments,
 	/* reset size summary */
 	file->read_size = 0;
 	file->write_size = 0;
+	file->uncompressed_size = 0;
 	INIT_FILE_CRC32(true, file->crc);
 
 	/* open backup mode file for read */
@@ -625,6 +627,8 @@ backup_data_file(backup_files_arg* arguments,
 				elog(ERROR, "Failed to read file \"%s\": %s",
 					 file->path, rc == PAGE_CHECKSUM_MISMATCH ? "data file checksum mismatch" : strerror(-rc));
 			n_blocks_read = rc;
+
+			file->uncompressed_size = (n_blocks_read - n_blocks_skipped)*BLCKSZ;
 		}
 		else
 		{
@@ -959,6 +963,7 @@ copy_file(fio_location from_location, const char *to_root,
 	/* reset size summary */
 	file->read_size = 0;
 	file->write_size = 0;
+	file->uncompressed_size = 0;
 
 	/* open backup mode file for read */
 	in = fio_fopen(file->path, PG_BINARY_R, from_location);
@@ -1046,6 +1051,9 @@ copy_file(fio_location from_location, const char *to_root,
 	}
 
 	file->write_size = (int64) file->read_size;
+
+	if (file->write_size > 0)
+		file->uncompressed_size = file->write_size;
 	/* finish CRC calculation and store into pgFile */
 	FIN_FILE_CRC32(true, crc);
 	file->crc = crc;

src/dir.c

@@ -1694,6 +1694,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_
 	file->crc = pgFileGetCRC(database_map_path, true, false,
 							 &file->read_size, FIO_BACKUP_HOST);
 	file->write_size = file->read_size;
+	file->uncompressed_size = file->read_size;
 	parray_append(backup_files_list, file);
 }
 

src/pg_probackup.h

@@ -140,6 +140,9 @@ typedef struct pgFile
 	int64	write_size;		/* size of the backed-up file. BYTES_INVALID means
 							   that the file existed but was not backed up
 							   because not modified since last backup. */
+	int64	uncompressed_size;	/* size of the backed-up file before compression
+								 * and adding block headers.
+								 */
 							/* we need int64 here to store '-1' value */
 	pg_crc32 crc;			/* CRC value of the file, regular file only */
 	char	*linked;		/* path of the linked file */
@@ -317,8 +320,17 @@ struct pgBackup
 	 * BYTES_INVALID means nothing was backed up.
 	 */
 	int64			data_bytes;
-	/* Size of WAL files needed to restore this backup */
+	/* Size of WAL files needed to replay on top of this
+	 * backup to reach the consistency.
+	 */
 	int64			wal_bytes;
+	/* Size of data files before applying compression and block header,
+	 * WAL files are not included.
+	 */
+	int64			uncompressed_bytes;
+
+	/* Size of data files in PGDATA at the moment of backup. */
+	int64			pgdata_bytes;
 
 	CompressAlg		compress_alg;
 	int				compress_level;

src/show.c

@@ -31,6 +31,7 @@ typedef struct ShowBackendRow
 	char		duration[20];
 	char		data_bytes[20];
 	char		wal_bytes[20];
+	char		zratio[20];
 	char		start_lsn[20];
 	char		stop_lsn[20];
 	const char *status;
@@ -385,6 +386,18 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup)
 		appendPQExpBuffer(buf, INT64_FORMAT, backup->wal_bytes);
 	}
 
+	if (backup->uncompressed_bytes >= 0)
+	{
+		json_add_key(buf, "uncompressed-bytes", json_level);
+		appendPQExpBuffer(buf, INT64_FORMAT, backup->uncompressed_bytes);
+	}
+
+	if (backup->uncompressed_bytes >= 0)
+	{
+		json_add_key(buf, "pgdata-bytes", json_level);
+		appendPQExpBuffer(buf, INT64_FORMAT, backup->pgdata_bytes);
+	}
+
 	if (backup->primary_conninfo)
 		json_add_value(buf, "primary_conninfo", backup->primary_conninfo,
 						json_level, true);
@@ -435,16 +448,16 @@ show_backup(const char *instance_name, time_t requested_backup_id)
 static void
 show_instance_plain(const char *instance_name, parray *backup_list, bool show_name)
 {
-#define SHOW_FIELDS_COUNT 13
+#define SHOW_FIELDS_COUNT 14
 	int			i;
 	const char *names[SHOW_FIELDS_COUNT] =
 					{ "Instance", "Version", "ID", "Recovery Time",
 					  "Mode", "WAL Mode", "TLI", "Time", "Data", "WAL",
-					  "Start LSN", "Stop LSN", "Status" };
+					  "Zratio", "Start LSN", "Stop LSN", "Status" };
 	const char *field_formats[SHOW_FIELDS_COUNT] =
 					{ " %-*s ", " %-*s ", " %-*s ", " %-*s ",
-					  " %-*s ", " %-*s ", " %-*s ", " %*s ", " %-*s ", " %-*s ",
-					  " %-*s ", " %-*s ", " %-*s "};
+					  " %-*s ", " %-*s ", " %-*s ", " %*s ", " %*s ", " %*s ",
+					  " %*s ", " %-*s ", " %-*s ", " %-*s "};
 	uint32		widths[SHOW_FIELDS_COUNT];
 	uint32		widths_sum = 0;
 	ShowBackendRow *rows;
@@ -465,6 +478,7 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na
 		pgBackup   *backup = parray_get(backup_list, i);
 		ShowBackendRow *row = &rows[i];
 		int			cur = 0;
+		float		zratio = 1;
 
 		/* Instance */
 		row->instance = instance_name;
@@ -503,7 +517,6 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na
 		cur++;
 
 		/* Current/Parent TLI */
-
 		if (backup->parent_backup_link != NULL)
 			parent_tli = backup->parent_backup_link->tli;
 
@@ -540,6 +553,19 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na
 		widths[cur] = Max(widths[cur], strlen(row->wal_bytes));
 		cur++;
 
+		/* Zratio (compression ratio) */
+		if (backup->uncompressed_bytes != BYTES_INVALID &&
+			(backup->uncompressed_bytes > 0 && backup->data_bytes > 0))
+		{
+			zratio = (float)backup->uncompressed_bytes / (backup->data_bytes);
+			snprintf(row->zratio, lengthof(row->zratio), "%.2f", zratio);
+		}
+		else
+			snprintf(row->zratio, lengthof(row->zratio), "%.2f", zratio);
+
+		widths[cur] = Max(widths[cur], strlen(row->zratio));
+		cur++;
+
 		/* Start LSN */
 		snprintf(row->start_lsn, lengthof(row->start_lsn), "%X/%X",
 				 (uint32) (backup->start_lsn >> 32),
@@ -630,6 +656,10 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na
 						  row->wal_bytes);
 		cur++;
 
+		appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur],
+						  row->zratio);
+		cur++;
+
 		appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur],
 						  row->start_lsn);
 		cur++;

src/util.c

@@ -400,6 +400,7 @@ copy_pgcontrol_file(const char *from_root, fio_location from_location,
 	file->crc = ControlFile.crc;
 	file->read_size = size;
 	file->write_size = size;
+	file->uncompressed_size = size;
 
 	join_path_components(to_path, to_root, file->rel_path);
 	writeControlFile(&ControlFile, to_path, to_location);

tests/archive.py

@@ -49,12 +49,12 @@ def test_pgpro434_1(self):
         node.slow_start()
 
         # Recreate backup catalog
+        self.clean_pb(backup_dir)
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
 
         # Make backup
-        self.backup_node(
-            backup_dir, 'node', node)
+        self.backup_node(backup_dir, 'node', node)
         node.cleanup()
 
         # Restore Database