Improvements for plain show, honest wal_bytes for STREAM backups and merged backups

Author: gsmolk

src/backup.c

@@ -553,6 +553,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo)
 		dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, 0,
 					  FIO_BACKUP_HOST);
 
+		/* TODO: Drop streamed WAL segments greater than stop_lsn */
 		for (i = 0; i < parray_num(xlog_files_list); i++)
 		{
 			pgFile	   *file = (pgFile *) parray_get(xlog_files_list, i);
@@ -702,7 +703,7 @@ do_backup(time_t start_time, bool no_validate)
 	}
 
 	elog(INFO, "Backup start, pg_probackup version: %s, instance: %s, backup ID: %s, backup mode: %s, "
-			"wal-method: %s, remote: %s, compress-algorithm: %s, compress-level: %i",
+			"wal mode: %s, remote: %s, compress-algorithm: %s, compress-level: %i",
 			PROGRAM_VERSION, instance_name, base36enc(start_time), pgBackupGetBackupMode(&current),
 			current.stream ? "STREAM" : "ARCHIVE", IsSshProtocol()  ? "true" : "false",
 			deparse_compress_alg(current.compress_alg), current.compress_level);

src/catalog.c

@@ -1121,6 +1121,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 	char		buf[BUFFERSZ];
 	size_t		write_len = 0;
 	int64 		backup_size_on_disk = 0;
+	int64 		wal_size_on_disk = 0;
 
 	pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST);
 	snprintf(path_temp, sizeof(path_temp), "%s.tmp", path);
@@ -1145,7 +1146,13 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 
 		/* Count the amount of the data actually copied */
 		if (S_ISREG(file->mode) && file->write_size > 0)
-			backup_size_on_disk += file->write_size;
+		{
+			/* TODO: in 3.0 add attribute is_walfile */
+			if (IsXLogFileName(file->name) && (file->external_dir_num == 0))
+				wal_size_on_disk += file->write_size;
+			else
+				backup_size_on_disk += file->write_size;
+		}
 
 		/* for files from PGDATA and external files use rel_path
 		 * streamed WAL files has rel_path relative not to "database/"
@@ -1227,6 +1234,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 
 	/* use extra variable to avoid reset of previous data_bytes value in case of error */
 	backup->data_bytes = backup_size_on_disk;
+	backup->wal_bytes = wal_size_on_disk;
 }
 
 /*

src/merge.c

@@ -345,32 +345,12 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup)
 	to_backup->merge_time = merge_time;
 	to_backup->end_time = time(NULL);
 
-	/*
-	 * Target backup must inherit wal mode too.
-	 */
+	/* Target backup must inherit wal mode too. */
 	to_backup->stream = from_backup->stream;
-	/* Compute summary of size of regular files in the backup */
-	to_backup->data_bytes = 0;
-	for (i = 0; i < parray_num(files); i++)
-	{
-		pgFile	   *file = (pgFile *) parray_get(files, i);
 
-		if (S_ISDIR(file->mode))
-		{
-			to_backup->data_bytes += 4096;
-			continue;
-		}
-		/* Count the amount of the data actually copied */
-		if (file->write_size > 0)
-			to_backup->data_bytes += file->write_size;
-	}
-	/* compute size of wal files of this backup stored in the archive */
+	/* ARCHIVE backup must inherit wal_bytes. */
 	if (!to_backup->stream)
-		to_backup->wal_bytes = instance_config.xlog_seg_size *
-			(to_backup->stop_lsn / instance_config.xlog_seg_size -
-			 to_backup->start_lsn / instance_config.xlog_seg_size + 1);
-	else
-		to_backup->wal_bytes = BYTES_INVALID;
+		to_backup->wal_bytes = from_backup->wal_bytes;
 
 	write_backup_filelist(to_backup, files, from_database_path, NULL);
 	write_backup(to_backup);

src/pg_probackup.h

@@ -314,7 +314,7 @@ struct pgBackup
 	 * BYTES_INVALID means nothing was backed up.
 	 */
 	int64			data_bytes;
-	/* Size of WAL files in archive needed to restore this backup */
+	/* Size of WAL files needed to restore this backup */
 	int64			wal_bytes;
 
 	CompressAlg		compress_alg;
@@ -812,7 +812,7 @@ int32  do_compress(void* dst, size_t dst_size, void const* src, size_t src_size,
 				   CompressAlg alg, int level, const char **errormsg);
 
 extern void pretty_size(int64 size, char *buf, size_t len);
-
+extern void pretty_time_interval(int64 num_seconds, char *buf, size_t len);
 
 extern PGconn *pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo);
 extern void check_system_identifiers(PGconn *conn, char *pgdata);

src/show.c

@@ -30,6 +30,7 @@ typedef struct ShowBackendRow
 	char		tli[20];
 	char		duration[20];
 	char		data_bytes[20];
+	char		wal_bytes[20];
 	char		start_lsn[20];
 	char		stop_lsn[20];
 	const char *status;
@@ -147,9 +148,15 @@ pretty_size(int64 size, char *buf, size_t len)
 	int64 	limit2 = limit * 2 - 1;
 
 	/* minus means the size is invalid */
-	if (size < 0)
+//	if (size < 0)
+//	{
+//		strncpy(buf, "----", len);
+//		return;
+//	}
+
+	if (size <= 0)
 	{
-		strncpy(buf, "----", len);
+		strncpy(buf, "0", len);
 		return;
 	}
 
@@ -180,6 +187,53 @@ pretty_size(int64 size, char *buf, size_t len)
 	}
 }
 
+void
+pretty_time_interval(int64 num_seconds, char *buf, size_t len)
+{
+	int 	seconds = 0;
+	int 	minutes = 0;
+	int 	hours = 0;
+	int 	days = 0;
+
+	if (num_seconds <= 0)
+	{
+		strncpy(buf, "0", len);
+		return;
+	}
+
+	days = num_seconds / (24 * 3600);
+	num_seconds %= (24 * 3600);
+
+	hours = num_seconds / 3600;
+	num_seconds %= 3600;
+
+	minutes = num_seconds / 60;
+	num_seconds %= 60;
+
+	seconds = num_seconds;
+
+	if (days > 0)
+	{
+		snprintf(buf, len, "%dd:%dh", days, hours);
+		return;
+	}
+
+	if (hours > 0)
+	{
+		snprintf(buf, len, "%dh:%dm", hours, minutes);
+		return;
+	}
+
+	if (minutes > 0)
+	{
+		snprintf(buf, len, "%dm:%ds", minutes, seconds);
+		return;
+	}
+
+	snprintf(buf, len, "%ds", seconds);
+	return;
+}
+
 /*
  * Initialize instance visualization.
  */
@@ -381,16 +435,16 @@ show_backup(const char *instance_name, time_t requested_backup_id)
 static void
 show_instance_plain(const char *instance_name, parray *backup_list, bool show_name)
 {
-#define SHOW_FIELDS_COUNT 12
+#define SHOW_FIELDS_COUNT 13
 	int			i;
 	const char *names[SHOW_FIELDS_COUNT] =
 					{ "Instance", "Version", "ID", "Recovery Time",
-					  "Mode", "WAL", "Current/Parent TLI", "Time", "Data",
+					  "Mode", "WAL Mode", "TLI", "Time", "Data", "WAL",
 					  "Start LSN", "Stop LSN", "Status" };
 	const char *field_formats[SHOW_FIELDS_COUNT] =
 					{ " %-*s ", " %-*s ", " %-*s ", " %-*s ",
-					  " %-*s ", " %-*s ", " %-*s ", " %*s ", " %*s ",
-					  " %*s ", " %*s ", " %-*s "};
+					  " %-*s ", " %-*s ", " %-*s ", " %*s ", " %-*s ", " %-*s ",
+					  " %-*s ", " %-*s ", " %-*s "};
 	uint32		widths[SHOW_FIELDS_COUNT];
 	uint32		widths_sum = 0;
 	ShowBackendRow *rows;
@@ -443,7 +497,7 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na
 		widths[cur] = Max(widths[cur], strlen(row->mode));
 		cur++;
 
-		/* WAL */
+		/* WAL mode*/
 		row->wal_mode = backup->stream ? "STREAM": "ARCHIVE";
 		widths[cur] = Max(widths[cur], strlen(row->wal_mode));
 		cur++;
@@ -453,22 +507,22 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na
 		if (backup->parent_backup_link != NULL)
 			parent_tli = backup->parent_backup_link->tli;
 
-		snprintf(row->tli, lengthof(row->tli), "%u / %u",
+		snprintf(row->tli, lengthof(row->tli), "%u/%u",
 				 backup->tli,
 				 backup->backup_mode == BACKUP_MODE_FULL ? 0 : parent_tli);
 		widths[cur] = Max(widths[cur], strlen(row->tli));
 		cur++;
 
 		/* Time */
 		if (backup->status == BACKUP_STATUS_RUNNING)
-			snprintf(row->duration, lengthof(row->duration), "%.*lfs", 0,
-					 difftime(current_time, backup->start_time));
+			pretty_time_interval(difftime(current_time, backup->start_time),
+							row->duration, lengthof(row->duration));
 		else if (backup->merge_time != (time_t) 0)
-			snprintf(row->duration, lengthof(row->duration), "%.*lfs", 0,
-					 difftime(backup->end_time, backup->merge_time));
+			pretty_time_interval(difftime(backup->end_time, backup->merge_time),
+							row->duration, lengthof(row->duration));
 		else if (backup->end_time != (time_t) 0)
-			snprintf(row->duration, lengthof(row->duration), "%.*lfs", 0,
-					 difftime(backup->end_time, backup->start_time));
+			pretty_time_interval(difftime(backup->end_time, backup->start_time),
+							row->duration, lengthof(row->duration));
 		else
 			StrNCpy(row->duration, "----", sizeof(row->duration));
 		widths[cur] = Max(widths[cur], strlen(row->duration));
@@ -480,6 +534,12 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na
 		widths[cur] = Max(widths[cur], strlen(row->data_bytes));
 		cur++;
 
+		/* WAL */
+		pretty_size(backup->wal_bytes, row->wal_bytes,
+					lengthof(row->wal_bytes));
+		widths[cur] = Max(widths[cur], strlen(row->wal_bytes));
+		cur++;
+
 		/* Start LSN */
 		snprintf(row->start_lsn, lengthof(row->start_lsn), "%X/%X",
 				 (uint32) (backup->start_lsn >> 32),
@@ -566,6 +626,10 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na
 						  row->data_bytes);
 		cur++;
 
+		appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur],
+						  row->wal_bytes);
+		cur++;
+
 		appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur],
 						  row->start_lsn);
 		cur++;