[Issue #143] Multi-timeline incremental chain

Author: gsmolk

doc/pgprobackup.xml

@@ -426,14 +426,6 @@ doc/src/sgml/pgprobackup.sgml
           or <application>libc</application>/<application>libicu</application> versions.
         </para>
       </listitem>
-      <listitem>
-        <para>
-          All backups in the incremental chain must belong to the same
-          timeline. For example, if you have taken incremental backups on a
-          standby server that gets promoted, you have to take another FULL
-          backup.
-        </para>
-      </listitem>
     </itemizedlist>
   </para>
  </refsect2>
@@ -753,9 +745,10 @@ ALTER ROLE backup WITH REPLICATION;
     <title>Setting up Continuous WAL Archiving</title>
     <para>
       Making backups in PAGE backup mode, performing
-      <link linkend="pbk-performing-point-in-time-pitr-recovery">PITR</link>
-      and making backups with
-      <link linkend="pbk-archive-mode">ARCHIVE</link> WAL delivery mode
+      <link linkend="pbk-performing-point-in-time-pitr-recovery">PITR</link>,
+      making backups with
+      <link linkend="pbk-archive-mode">ARCHIVE</link> WAL delivery mode and
+      running incremental backup after timeline switch
       require
       <ulink url="https://postgrespro.com/docs/postgresql/current/continuous-archiving.html">continuous
       WAL archiving</ulink> to be enabled. To set up continuous

src/backup.c

@@ -153,6 +153,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 	PGconn	   *master_conn = NULL;
 	PGconn	   *pg_startbackup_conn = NULL;
 
+	/* used for multitimeline incremental backup */
+	parray       *tli_list = NULL;
+
+
 	/* for fancy reporting */
 	time_t		start_time, end_time;
 	char		pretty_time[20];
@@ -181,17 +185,43 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 		current.backup_mode == BACKUP_MODE_DIFF_PTRACK ||
 		current.backup_mode == BACKUP_MODE_DIFF_DELTA)
 	{
-		char		prev_backup_filelist_path[MAXPGPATH];
-
 		/* get list of backups already taken */
 		backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID);
 
 		prev_backup = catalog_get_last_data_backup(backup_list, current.tli, current.start_time);
 		if (prev_backup == NULL)
-			elog(ERROR, "Valid backup on current timeline %X is not found. "
-						"Create new FULL backup before an incremental one.",
+		{
+			/* try to setup multi-timeline backup chain */
+			elog(WARNING, "Valid backup on current timeline %u is not found, "
+						"try to look up on previous timelines",
 						current.tli);
 
+			tli_list = catalog_get_timelines(&instance_config);
+
+			if (parray_num(tli_list) == 0)
+				elog(WARNING, "Cannot find valid backup on previous timelines, "
+							"WAL archive is not available");
+			else
+			{
+				prev_backup = get_multi_timeline_parent(backup_list, tli_list, current.tli,
+														current.start_time, &instance_config);
+
+				if (prev_backup == NULL)
+					elog(WARNING, "Cannot find valid backup on previous timelines");
+			}
+
+			/* failed to find suitable parent, error out */
+			if (!prev_backup)
+				elog(ERROR, "Create new full backup before an incremental one");
+		}
+	}
+
+	if (prev_backup)
+	{
+		char		prev_backup_filelist_path[MAXPGPATH];
+
+		elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time));
+
 		join_path_components(prev_backup_filelist_path, prev_backup->root_dir,
 															DATABASE_FILE_LIST);
 		/* Files of previous backup needed by DELTA backup */
@@ -378,8 +408,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 	if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
 		current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
 	{
-		elog(INFO, "Compiling pagemap of changed blocks");
+		bool pagemap_isok = true;
+
 		time(&start_time);
+		elog(INFO, "Extracting pagemap of changed blocks");
 
 		if (current.backup_mode == BACKUP_MODE_DIFF_PAGE)
 		{
@@ -388,8 +420,9 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 			 * reading WAL segments present in archives up to the point
 			 * where this backup has started.
 			 */
-			extractPageMap(arclog_path, current.tli, instance_config.xlog_seg_size,
-						   prev_backup->start_lsn, current.start_lsn);
+			pagemap_isok = extractPageMap(arclog_path, instance_config.xlog_seg_size,
+						   prev_backup->start_lsn, prev_backup->tli,
+						   current.start_lsn, current.tli, tli_list);
 		}
 		else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
 		{
@@ -407,8 +440,14 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 		}
 
 		time(&end_time);
-		elog(INFO, "Pagemap compiled, time elapsed %.0f sec",
-			 difftime(end_time, start_time));
+
+		/* TODO: add ms precision */
+		if (pagemap_isok)
+			elog(INFO, "Pagemap successfully extracted, time elapsed %.0f sec",
+				 difftime(end_time, start_time));
+		else
+			elog(ERROR, "Pagemap extraction failed, time elasped: %.0f sec",
+				 difftime(end_time, start_time));
 	}
 
 	/*
@@ -667,6 +706,15 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 		elog(INFO, "Backup files are synced, time elapsed: %s", pretty_time);
 	}
 
+	/* be paranoid about instance been from the past */
+	if (current.backup_mode != BACKUP_MODE_FULL &&
+		current.stop_lsn < prev_backup->stop_lsn)
+			elog(ERROR, "Current backup STOP LSN %X/%X is lower than STOP LSN %X/%X of previous backup %s. "
+				"It may indicate that we are trying to backup PostgreSQL instance from the past.",
+				(uint32) (current.stop_lsn >> 32), (uint32) (current.stop_lsn),
+				(uint32) (prev_backup->stop_lsn >> 32), (uint32) (prev_backup->stop_lsn),
+				base36enc(prev_backup->stop_lsn));
+
 	/* clean external directories list */
 	if (external_dirs)
 		free_dir_list(external_dirs);
@@ -678,6 +726,12 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 		parray_free(backup_list);
 	}
 
+	if (tli_list)
+	{
+		parray_walk(tli_list, timelineInfoFree);
+		parray_free(tli_list);
+	}
+
 	parray_walk(backup_files_list, pgFileFree);
 	parray_free(backup_files_list);
 	backup_files_list = NULL;

src/catalog.c

@@ -42,6 +42,24 @@ timelineInfoNew(TimeLineID tli)
 	return tlinfo;
 }
 
+/* free timelineInfo object */
+void
+timelineInfoFree(void *tliInfo)
+{
+	timelineInfo *tli = (timelineInfo *) tliInfo;
+
+	parray_walk(tli->xlog_filelist, pgFileFree);
+	parray_free(tli->xlog_filelist);
+
+	if (tli->backups)
+	{
+		parray_walk(tli->backups, pgBackupFree);
+		parray_free(tli->backups);
+	}
+
+	pfree(tliInfo);
+}
+
 /* Iterate over locked backups and delete locks files */
 static void
 unlink_lock_atexit(void)
@@ -621,11 +639,7 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current
 					 * anomalies.
 					 */
 					if (is_parent(full_backup->start_time, backup, true))
-					{
-						elog(INFO, "Parent backup: %s",
-							base36enc(backup->start_time));
 						return backup;
-					}
 			}
 		}
 		/* skip yourself */
@@ -641,6 +655,149 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current
 	return NULL;
 }
 
+/*
+ * For multi-timeline chain, look up suitable parent for incremental backup.
+ * Multi-timeline chain has full backup and one or more descendants located
+ * on different timelines.
+ */
+pgBackup *
+get_multi_timeline_parent(parray *backup_list, parray *tli_list,
+	                      TimeLineID current_tli, time_t current_start_time,
+						  InstanceConfig *instance)
+{
+	int           i;
+	timelineInfo *my_tlinfo = NULL;
+	timelineInfo *tmp_tlinfo = NULL;
+	pgBackup     *ancestor_backup = NULL;
+
+	/* there are no timelines in the archive */
+	if (parray_num(tli_list) == 0)
+		return NULL;
+
+	/* look for current timelineInfo */
+	for (i = 0; i < parray_num(tli_list); i++)
+	{
+		timelineInfo  *tlinfo = (timelineInfo  *) parray_get(tli_list, i);
+
+		if (tlinfo->tli == current_tli)
+		{
+			my_tlinfo = tlinfo;
+			break;
+		}
+	}
+
+	if (my_tlinfo == NULL)
+		return NULL;
+
+	/* Locate tlinfo of suitable full backup.
+	 * Consider this example:
+	 *  t3                    s2-------X <-! We are here
+	 *                        /
+	 *  t2         s1----D---*----E--->
+	 *             /
+	 *  t1--A--B--*---C------->
+	 *
+	 * A, E - full backups
+	 * B, C, D - incremental backups
+	 *
+	 * We must find A.
+	 */
+	tmp_tlinfo = my_tlinfo;
+	while (tmp_tlinfo->parent_link)
+	{
+		/* if timeline has backups, iterate over them */
+		if (tmp_tlinfo->parent_link->backups)
+		{
+			for (i = 0; i < parray_num(tmp_tlinfo->parent_link->backups); i++)
+			{
+				pgBackup *backup = (pgBackup *) parray_get(tmp_tlinfo->parent_link->backups, i);
+
+				if (backup->backup_mode == BACKUP_MODE_FULL &&
+					(backup->status == BACKUP_STATUS_OK ||
+					 backup->status == BACKUP_STATUS_DONE) &&
+					 backup->stop_lsn <= tmp_tlinfo->switchpoint)
+				{
+					ancestor_backup = backup;
+					break;
+				}
+			}
+		}
+
+		if (ancestor_backup)
+			break;
+
+		tmp_tlinfo = tmp_tlinfo->parent_link;
+	}
+
+	/* failed to find valid FULL backup on parent timelines */
+	if (!ancestor_backup)
+		return NULL;
+	else
+		elog(LOG, "Latest valid full backup: %s, tli: %i",
+			base36enc(ancestor_backup->start_time), ancestor_backup->tli);
+
+	/* At this point we found suitable full backup,
+	 * now we must find his latest child, suitable to be
+	 * parent of current incremental backup.
+	 * Consider this example:
+	 *  t3                    s2-------X <-! We are here
+	 *                        /
+	 *  t2         s1----D---*----E--->
+	 *             /
+	 *  t1--A--B--*---C------->
+	 *
+	 * A, E - full backups
+	 * B, C, D - incremental backups
+	 *
+	 * We found A, now we must find D.
+	 */
+
+	/* Optimistically, look on current timeline for valid incremental backup, child of ancestor */
+	if (my_tlinfo->backups)
+	{
+		for (i = 0; i < parray_num(my_tlinfo->backups); i++)
+		{
+			pgBackup *tmp_backup = NULL;
+			pgBackup *backup = (pgBackup *) parray_get(my_tlinfo->backups, i);
+
+			/* found suitable parent */
+			if (scan_parent_chain(backup, &tmp_backup) == 2 &&
+				is_parent(ancestor_backup->start_time, backup, false))
+				return backup;
+		}
+	}
+
+	/* Iterate over parent timelines and look for a valid backup, child of ancestor */
+	tmp_tlinfo = my_tlinfo;
+	while (tmp_tlinfo->parent_link)
+	{
+
+		/* if timeline has backups, iterate over them */
+		if (tmp_tlinfo->parent_link->backups)
+		{
+			for (i = 0; i < parray_num(tmp_tlinfo->parent_link->backups); i++)
+			{
+				pgBackup *tmp_backup = NULL;
+				pgBackup *backup = (pgBackup *) parray_get(tmp_tlinfo->parent_link->backups, i);
+
+				/* We are not interested in backups
+				 * located outside of our timeline history
+				 */
+				if (backup->stop_lsn > tmp_tlinfo->switchpoint)
+					continue;
+
+				if (scan_parent_chain(backup, &tmp_backup) == 2 &&
+					is_parent(ancestor_backup->start_time, backup, true))
+					return backup;
+			}
+		}
+
+		tmp_tlinfo = tmp_tlinfo->parent_link;
+	}
+
+	return NULL;
+}
+
 /* create backup directory in $BACKUP_PATH */
 int
 pgBackupCreateDir(pgBackup *backup)