Merge branch 'REL_2_5'

Author: postgres-dev

doc/pgprobackup.xml

@@ -811,7 +811,7 @@ msgstr ""
 #: src/help.c:360 src/help.c:521 src/help.c:588 src/help.c:635 src/help.c:715
 #: src/help.c:761 src/help.c:833
 #, c-format
-msgid "                                   directory for file logging (default: BACKUP_PATH/log)\n"
+msgid "                                   directory for file logging (default: BACKUP_DIR/log)\n"
 msgstr ""
 
 #: src/help.c:361 src/help.c:522 src/help.c:589 src/help.c:636 src/help.c:716

po/ru.po

@@ -113,7 +113,7 @@ static parray *setup_push_filelist(const char *archive_status_dir,
  * set archive_command to
  * 'pg_probackup archive-push -B /home/anastasia/backup --wal-file-name %f',
  * to move backups into arclog_path.
- * Where archlog_path is $BACKUP_PATH/wal/instance_name
+ * Where archlog_path is $BACKUP_DIR/wal/instance_name
  */
 void
 do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir,
@@ -1126,7 +1126,7 @@ do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const cha
 	join_path_components(absolute_wal_file_path, current_dir, wal_file_path);
 
 	/* full filepath to WAL file in archive directory.
-	 * $BACKUP_PATH/wal/instance_name/000000010000000000000001 */
+	 * $BACKUP_DIR/wal/instance_name/000000010000000000000001 */
 	join_path_components(backup_wal_file_path, instanceState->instance_wal_subdir_path, wal_file_name);
 
 	INSTR_TIME_SET_CURRENT(start_time);

src/archive.c

@@ -122,6 +122,8 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
 	char		pretty_time[20];
 	char		pretty_bytes[20];
 
+	pgFile	*src_pg_control_file = NULL;
+
 	elog(INFO, "Database backup start");
 	if(current.external_dir_str)
 	{
@@ -424,6 +426,24 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
 
 	}
 
+	/*
+	 * find pg_control file
+	 * We'll copy it last
+	 */
+	{
+		int control_file_elem_index;
+		pgFile search_key;
+		MemSet(&search_key, 0, sizeof(pgFile));
+		/* pgFileCompareRelPathWithExternal uses only .rel_path and .external_dir_num for comparision */
+		search_key.rel_path = XLOG_CONTROL_FILE;
+		search_key.external_dir_num = 0;
+		control_file_elem_index = parray_bsearch_index(backup_files_list, &search_key, pgFileCompareRelPathWithExternal);
+
+		if (control_file_elem_index < 0)
+			elog(ERROR, "File \"%s\" not found in PGDATA %s", XLOG_CONTROL_FILE, current.database_dir);
+		src_pg_control_file = (pgFile *)parray_get(backup_files_list, control_file_elem_index);
+	}
+
 	/* setup thread locks */
 	pfilearray_clear_locks(backup_files_list);
 
@@ -483,6 +503,26 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
 			backup_isok = false;
 	}
 
+	/* copy pg_control at very end */
+	if (backup_isok)
+	{
+
+		elog(progress ? INFO : LOG, "Progress: Backup file \"%s\"",
+			 src_pg_control_file->rel_path);
+
+		char	from_fullpath[MAXPGPATH];
+		char	to_fullpath[MAXPGPATH];
+		join_path_components(from_fullpath, instance_config.pgdata, src_pg_control_file->rel_path);
+		join_path_components(to_fullpath, current.database_dir, src_pg_control_file->rel_path);
+
+		backup_non_data_file(src_pg_control_file, NULL,
+					 from_fullpath, to_fullpath,
+					 current.backup_mode, current.parent_backup,
+					 true);
+	}
+
+
+
 	time(&end_time);
 	pretty_time_interval(difftime(end_time, start_time),
 						 pretty_time, lengthof(pretty_time));
@@ -510,17 +550,8 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
 	{
 		pgFile	   *pg_control = NULL;
 
-		for (i = 0; i < parray_num(backup_files_list); i++)
-		{
-			pgFile	   *tmp_file = (pgFile *) parray_get(backup_files_list, i);
+		pg_control = src_pg_control_file;
 
-			if (tmp_file->external_dir_num == 0 &&
-				(strcmp(tmp_file->rel_path, XLOG_CONTROL_FILE) == 0))
-			{
-				pg_control = tmp_file;
-				break;
-			}
-		}
 
 		if (!pg_control)
 			elog(ERROR, "Failed to find file \"%s\" in backup filelist.",
@@ -2076,6 +2107,13 @@ backup_files(void *arg)
 		/* We have already copied all directories */
 		if (S_ISDIR(file->mode))
 			continue;
+		/*
+		 * Don't copy the pg_control file now, we'll copy it last
+		 */
+		if(file->external_dir_num == 0 && pg_strcasecmp(file->rel_path, XLOG_CONTROL_FILE) == 0)
+		{
+			continue;
+		}
 
 		if (arguments->thread_num == 1)
 		{

src/backup.c

@@ -1437,7 +1437,7 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list,
 }
 
 /*
- * Create backup directory in $BACKUP_PATH
+ * Create backup directory in $BACKUP_DIR
  * (with proposed backup->backup_id)
  * and initialize this directory.
  * If creation of directory fails, then

src/catalog.c

@@ -171,10 +171,13 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn,
 
 		if (current.backup_mode != BACKUP_MODE_FULL)
 		{
-			dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST, false);
+			ControlFileData dst_control;
+			get_control_file_or_back_file(dest_pgdata, FIO_LOCAL_HOST, &dst_control);
+			dest_id = dst_control.system_identifier;
+
 			if (source_conn_id != dest_id)
-			elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu",
-				source_conn_id, dest_pgdata, dest_id);
+			elog(ERROR, "Database identifiers mismatch: we connected to DB id %llu, but in \"%s\" we found id %llu",
+				(long long)source_conn_id, dest_pgdata, (long long)dest_id);
 		}
 	}
 
@@ -640,6 +643,9 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
 	ssize_t		transfered_walfiles_bytes = 0;
 	char		pretty_source_bytes[20];
 
+	char	dest_pg_control_fullpath[MAXPGPATH];
+	char	dest_pg_control_bak_fullpath[MAXPGPATH];
+
 	source_conn = catchup_init_state(&source_node_info, source_pgdata, dest_pgdata);
 	catchup_preflight_checks(&source_node_info, source_conn, source_pgdata, dest_pgdata);
 
@@ -935,6 +941,9 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
 			Assert(file->external_dir_num == 0);
 			if (pg_strcasecmp(file->name, RELMAPPER_FILENAME) == 0)
 				redundant = true;
+			/* global/pg_control.pbk.bak is always keeped, because it's needed for restart failed incremental restore */
+			if (pg_strcasecmp(file->rel_path, XLOG_CONTROL_BAK_FILE) == 0)
+				redundant = false;
 
 			/* if file does not exists in destination list, then we can safely unlink it */
 			if (redundant)
@@ -966,6 +975,28 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
 	if (dest_filelist)
 		parray_qsort(dest_filelist, pgFileCompareRelPathWithExternal);
 
+	join_path_components(dest_pg_control_fullpath, dest_pgdata, XLOG_CONTROL_FILE);
+	join_path_components(dest_pg_control_bak_fullpath, dest_pgdata, XLOG_CONTROL_BAK_FILE);
+	/*
+	 * rename (if it exist) dest control file before restoring
+	 * if it doesn't exist, that mean, that we already restoring in a previously failed
+	 * pgdata, where XLOG_CONTROL_BAK_FILE exist
+	 */
+	if (current.backup_mode != BACKUP_MODE_FULL && !dry_run)
+	{
+		if (!fio_access(dest_pg_control_fullpath, F_OK, FIO_LOCAL_HOST))
+		{
+			pgFile *dst_control;
+			dst_control = pgFileNew(dest_pg_control_bak_fullpath, XLOG_CONTROL_BAK_FILE,
+			true,0, FIO_BACKUP_HOST);
+
+			if(!fio_access(dest_pg_control_bak_fullpath, F_OK, FIO_LOCAL_HOST))
+				fio_delete(dst_control->mode, dest_pg_control_bak_fullpath, FIO_LOCAL_HOST);
+			fio_rename(dest_pg_control_fullpath, dest_pg_control_bak_fullpath, FIO_LOCAL_HOST);
+			pgFileFree(dst_control);
+		}
+	}
+
 	/* run copy threads */
 	elog(INFO, "Start transferring data files");
 	time(&start_time);
@@ -985,6 +1016,15 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
 		copy_pgcontrol_file(from_fullpath, FIO_DB_HOST,
 				to_fullpath, FIO_LOCAL_HOST, source_pg_control_file);
 		transfered_datafiles_bytes += source_pg_control_file->size;
+
+		/* Now backup control file can be deled */
+		if (current.backup_mode != BACKUP_MODE_FULL && !fio_access(dest_pg_control_bak_fullpath, F_OK, FIO_LOCAL_HOST)){
+			pgFile *dst_control;
+			dst_control = pgFileNew(dest_pg_control_bak_fullpath, XLOG_CONTROL_BAK_FILE,
+			true,0, FIO_BACKUP_HOST);
+			fio_delete(dst_control->mode, dest_pg_control_bak_fullpath, FIO_LOCAL_HOST);
+			pgFileFree(dst_control);
+		}
 	}
 
 	if (!catchup_isok && !dry_run)

src/catchup.c

@@ -269,18 +269,21 @@ static const char *current_group = NULL;
  * Show configure options including default values.
  */
 void
-do_show_config(void)
+do_show_config(bool show_base_units)
 {
 	int			i;
 
 	show_configure_start();
 
 	for (i = 0; instance_options[i].type; i++)
 	{
+		if (show_base_units && strchr("bBiIuU", instance_options[i].type) && instance_options[i].get_value == *option_get_value)
+			instance_options[i].flags |= GET_VAL_IN_BASE_UNITS;	/* Set flag */
 		if (show_format == SHOW_PLAIN)
 			show_configure_plain(&instance_options[i]);
 		else
 			show_configure_json(&instance_options[i]);
+		instance_options[i].flags &= ~(GET_VAL_IN_BASE_UNITS); /* Reset flag. It was resetted in option_get_value(). Probably this reset isn't needed */
 	}
 
 	show_configure_end();
@@ -801,6 +804,6 @@ show_configure_json(ConfigOption *opt)
 		return;
 
 	json_add_value(&show_buf, opt->lname, value, json_level,
-				   true);
+				   !(opt->flags & GET_VAL_IN_BASE_UNITS));
 	pfree(value);
 }

src/configure.c

@@ -2490,7 +2490,10 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map,
 			file->rel_path, file->hdr_off, z_len, file->hdr_crc);
 
 	if (fwrite(zheaders, 1, z_len, hdr_map->fp) != z_len)
+	{
+		pthread_mutex_unlock(&(hdr_map->mutex));
 		elog(ERROR, "Cannot write to file \"%s\": %s", map_path, strerror(errno));
+	}
 
 	file->hdr_size = z_len;	  /* save the length of compressed headers */
 	hdr_map->offset += z_len; /* update current offset in map */

src/data.c

@@ -1867,4 +1867,4 @@ set_forkname(pgFile *file)
 	file->segno = segno;
 	file->is_datafile = file->forkName == none;
 	return true;
-}
+}