Merge branch 'master' into stable

Author: za-arthur

src/parsexlog.c

@@ -305,6 +305,8 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli,
 		/* By default there is some error */
 		thread_args[i].ret = 1;
 
+		threads_need++;
+
 		/* Adjust startpoint to the next thread */
 		if (nextSegNoToRead == 0)
 			XLByteToSeg(startpoint, nextSegNoToRead);
@@ -317,10 +319,6 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli,
 		if (nextSegNoToRead > endSegNo)
 			break;
 		XLogSegNoOffsetToRecPtr(nextSegNoToRead, 0, startpoint);
-		/* Skip over the page header */
-		startpoint += SizeOfXLogLongPHD;
-
-		threads_need++;
 	}
 
 	/* Run threads */

tests/merge.py

@@ -375,7 +375,8 @@ def test_merge_ptrack_truncate(self):
                 'wal_level': 'replica',
                 'max_wal_senders': '2',
                 'checkpoint_timeout': '300s',
-                'autovacuum': 'off'
+                'autovacuum': 'off',
+                'ptrack_enable': 'on'
             }
         )
         node_restored = self.make_simple_node(
@@ -410,7 +411,7 @@ def test_merge_ptrack_truncate(self):
             "vacuum t_heap")
 
         self.backup_node(
-            backup_dir, 'node', node, backup_type='delta')
+            backup_dir, 'node', node, backup_type='ptrack')
 
         if self.paranoia:
             pgdata = self.pgdata_content(node.data_dir)
@@ -452,3 +453,92 @@ def test_merge_ptrack_truncate(self):
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    def test_merge_delta_delete(self):
+        """
+        Make node, create tablespace with table, take full backup,
+        alter tablespace location, take delta backup, restore database.
+        """
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
+            set_replication=True, initdb_params=['--data-checksums'],
+            pg_options={
+                'wal_level': 'replica',
+                'max_wal_senders': '2',
+                'checkpoint_timeout': '30s',
+                'autovacuum': 'off'
+            }
+        )
+
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.start()
+
+        self.create_tblspace_in_node(node, 'somedata')
+
+        # FULL backup
+        self.backup_node(backup_dir, 'node', node, options=["--stream"])
+
+        node.safe_psql(
+            "postgres",
+            "create table t_heap tablespace somedata as select i as id,"
+            " md5(i::text) as text, md5(i::text)::tsvector as tsvector"
+            " from generate_series(0,100) i"
+        )
+
+        node.safe_psql(
+            "postgres",
+            "delete from t_heap"
+        )
+
+        node.safe_psql(
+            "postgres",
+            "vacuum t_heap"
+        )
+
+        # DELTA BACKUP
+        self.backup_node(
+            backup_dir, 'node', node,
+            backup_type='delta',
+            options=["--stream"]
+        )
+
+        if self.paranoia:
+            pgdata = self.pgdata_content(node.data_dir)
+
+        backup_id = self.show_pb(backup_dir, "node")[1]["id"]
+        self.merge_backup(backup_dir, "node", backup_id)
+
+        # RESTORE
+        node_restored = self.make_simple_node(
+            base_dir="{0}/{1}/node_restored".format(module_name, fname)
+        )
+        node_restored.cleanup()
+
+        self.restore_node(
+            backup_dir, 'node', node_restored,
+            options=[
+                "-j", "4",
+                "-T", "{0}={1}".format(
+                    self.get_tblspace_path(node, 'somedata'),
+                    self.get_tblspace_path(node_restored, 'somedata')
+                )
+            ]
+        )
+
+        # GET RESTORED PGDATA AND COMPARE
+        if self.paranoia:
+            pgdata_restored = self.pgdata_content(node_restored.data_dir)
+            self.compare_pgdata(pgdata, pgdata_restored)
+
+        # START RESTORED NODE
+        node_restored.append_conf(
+            'postgresql.auto.conf', 'port = {0}'.format(node_restored.port))
+        node_restored.start()
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)

tests/page.py

@@ -149,7 +149,10 @@ def test_page_stream(self):
         page_result = node.execute("postgres", "SELECT * FROM t_heap")
         page_backup_id = self.backup_node(
             backup_dir, 'node', node,
-            backup_type='page', options=['--stream'])
+            backup_type='page', options=['--stream', '-j', '4'])
+
+        if self.paranoia:
+            pgdata = self.pgdata_content(node.data_dir)
 
         # Drop Node
         node.cleanup()
@@ -162,6 +165,12 @@ def test_page_stream(self):
                 backup_id=full_backup_id, options=["-j", "4"]),
             '\n Unexpected Error Message: {0}\n'
             ' CMD: {1}'.format(repr(self.output), self.cmd))
+
+        # GET RESTORED PGDATA AND COMPARE
+        if self.paranoia:
+            pgdata_restored = self.pgdata_content(node.data_dir)
+            self.compare_pgdata(pgdata, pgdata_restored)
+
         node.slow_start()
         full_result_new = node.execute("postgres", "SELECT * FROM t_heap")
         self.assertEqual(full_result, full_result_new)
@@ -211,7 +220,7 @@ def test_page_archive(self):
         node.safe_psql(
             "postgres",
             "create table t_heap as select i as id, md5(i::text) as text, "
-            "md5(i::text)::tsvector as tsvector from generate_series(0,1) i")
+            "md5(i::text)::tsvector as tsvector from generate_series(0,100) i")
         full_result = node.execute("postgres", "SELECT * FROM t_heap")
         full_backup_id = self.backup_node(
             backup_dir, 'node', node, backup_type='full')
@@ -221,10 +230,14 @@ def test_page_archive(self):
             "postgres",
             "insert into t_heap select i as id, "
             "md5(i::text) as text, md5(i::text)::tsvector as tsvector "
-            "from generate_series(0,2) i")
+            "from generate_series(100, 200) i")
         page_result = node.execute("postgres", "SELECT * FROM t_heap")
         page_backup_id = self.backup_node(
-            backup_dir, 'node', node, backup_type='page')
+            backup_dir, 'node', node,
+            backup_type='page', options=["-j", "4"])
+
+        if self.paranoia:
+            pgdata = self.pgdata_content(node.data_dir)
 
         # Drop Node
         node.cleanup()
@@ -241,6 +254,7 @@ def test_page_archive(self):
                     "--recovery-target-action=promote"]),
             '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
                 repr(self.output), self.cmd))
+
         node.slow_start()
 
         full_result_new = node.execute("postgres", "SELECT * FROM t_heap")
@@ -259,6 +273,12 @@ def test_page_archive(self):
                     "--recovery-target-action=promote"]),
             '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
                 repr(self.output), self.cmd))
+
+         # GET RESTORED PGDATA AND COMPARE
+        if self.paranoia:
+            pgdata_restored = self.pgdata_content(node.data_dir)
+            self.compare_pgdata(pgdata, pgdata_restored)
+
         node.slow_start()
 
         page_result_new = node.execute("postgres", "SELECT * FROM t_heap")