tests: fixes

Author: gsmolk

tests/__init__.py

@@ -12,6 +12,7 @@
 
 def load_tests(loader, tests, pattern):
     suite = unittest.TestSuite()
+#    suite.addTests(loader.loadTestsFromModule(auth_test))
     suite.addTests(loader.loadTestsFromModule(archive))
     suite.addTests(loader.loadTestsFromModule(backup_test))
 #    suite.addTests(loader.loadTestsFromModule(cfs_backup))
@@ -42,7 +43,6 @@ def load_tests(loader, tests, pattern):
     suite.addTests(loader.loadTestsFromModule(validate_test))
     suite.addTests(loader.loadTestsFromModule(pgpro560))
     suite.addTests(loader.loadTestsFromModule(pgpro589))
-    suite.addTests(loader.loadTestsFromModule(auth_test))
 
     return suite
 

tests/archive.py

@@ -382,3 +382,38 @@ def test_master_and_replica_concurrent_archiving(self):
         backup_id = self.backup_node(backup_dir, 'master', master)
         self.validate_pb(backup_dir, 'master')
         self.assertEqual('OK', self.show_pb(backup_dir, 'master', backup_id)['status'])
+
+    # @unittest.expectedFailure
+    @unittest.skip("skip")
+    def test_archive_compress(self):
+        """Description in jira issue PGPRO-434"""
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+            set_replication=True,
+            initdb_params=['--data-checksums'],
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s'}
+            )
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        # force more frequent wal switch
+        node.append_conf('postgresql.auto.conf', 'archive_timeout  = 30')
+        node.start()
+
+        node.safe_psql(
+            "postgres",
+            "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,100) i")
+
+        result = node.safe_psql("postgres", "SELECT * FROM t_heap")
+        self.backup_node(backup_dir, 'node', node)
+
+        node.cleanup()
+
+        self.restore_node(backup_dir, 'node', node)
+        node.start()
+
+        self.assertEqual(result, node.safe_psql("postgres", "SELECT * FROM t_heap"),
+            'data after restore not equal to original data')
+        # Clean after yourself
+        # self.del_test_dir(module_name, fname)

tests/backup_test.py

@@ -113,9 +113,10 @@ def test_incremental_backup_without_full(self):
             self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format(
                 repr(self.output), self.cmd))
         except ProbackupException as e:
-            self.assertEqual(e.message,
-                'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n',
-                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
+            self.assertIn(
+                "ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.",
+                e.message,
+                "\n Unexpected Error Message: {0}\n CMD: {1}".format(repr(e.message), self.cmd))
 
         sleep(1)
 
@@ -125,9 +126,10 @@ def test_incremental_backup_without_full(self):
             self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format(
                 repr(self.output), self.cmd))
         except ProbackupException as e:
-            self.assertEqual(e.message,
-                'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n',
-                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
+            self.assertIn(
+                "ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.",
+                e.message,
+                "\n Unexpected Error Message: {0}\n CMD: {1}".format(repr(e.message), self.cmd))
 
         self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR")
 
@@ -170,8 +172,9 @@ def test_incremental_backup_corrupt_full(self):
             self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format(
                 repr(self.output), self.cmd))
         except ProbackupException as e:
-            self.assertEqual(e.message,
-                "ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n",
+            self.assertIn(
+                "ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.",
+                e.message,
                 "\n Unexpected Error Message: {0}\n CMD: {1}".format(repr(e.message), self.cmd))
 
         # sleep(1)

tests/expected/option_help.out

@@ -5,10 +5,11 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
 
   pg_probackup version
 
-  pg_probackup init -B backup-path [-l]
+  pg_probackup init -B backup-path
 
   pg_probackup set-config -B backup-dir --instance=instance_name
-                 [--log-level=log-level]
+                 [--log-level-console=log-level-console]
+                 [--log-level-file=log-level-file]
                  [--log-filename=log-filename]
                  [--error-log-filename=error-log-filename]
                  [--log-directory=log-directory]
@@ -26,7 +27,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
   pg_probackup show-config -B backup-dir --instance=instance_name
 
   pg_probackup backup -B backup-path -b backup-mode --instance=instance_name
-                 [-C] [-l] [--stream [-S slot-name]] [--backup-pg-log]
+                 [-C] [--stream [-S slot-name]] [--backup-pg-log]
                  [-j num-threads] [--archive-timeout=archive-timeout]
                  [--compress-algorithm=compress-algorithm]
                  [--compress-level=compress-level]
@@ -37,20 +38,20 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
                  [--replica-timeout=timeout]
 
   pg_probackup restore -B backup-dir --instance=instance_name
-                 [-D pgdata-dir] [-l] [-i backup-id] [--progress]
+                 [-D pgdata-dir] [-i backup-id] [--progress]
                  [--time=time|--xid=xid [--inclusive=boolean]]
                  [--timeline=timeline] [-T OLDDIR=NEWDIR]
 
   pg_probackup validate -B backup-dir [--instance=instance_name]
-                 [-i backup-id] [-l] [--progress]
+                 [-i backup-id] [--progress]
                  [--time=time|--xid=xid [--inclusive=boolean]]
                  [--timeline=timeline]
 
   pg_probackup show -B backup-dir
                  [--instance=instance_name [-i backup-id]]
 
   pg_probackup delete -B backup-dir --instance=instance_name
-                 [--wal] [-i backup-id | --expired] [-l]
+                 [--wal] [-i backup-id | --expired]
 
   pg_probackup add-instance -B backup-dir -D pgdata-dir
                  --instance=instance_name

tests/expected/option_version.out

@@ -1 +1 @@
-pg_probackup 2.0.13
+pg_probackup 2.0.13

tests/helpers/ptrack_helpers.py

@@ -559,7 +559,7 @@ def get_recovery_conf(self, node):
                 out_dict[key.strip()] = value.strip(" '").replace("'\n", "")
         return out_dict
 
-    def set_archiving(self, backup_dir, instance, node, replica=False):
+    def set_archiving(self, backup_dir, instance, node, replica=False, compress=False):
 
         if replica:
             archive_mode = 'always'
@@ -576,7 +576,13 @@ def set_archiving(self, backup_dir, instance, node, replica=False):
                 "archive_mode = {0}".format(archive_mode)
                 )
         if os.name == 'posix':
-            node.append_conf(
+            if compress:
+                node.append_conf(
+                    "postgresql.auto.conf",
+                    "archive_command = '{0} archive-push -B {1} --instance={2} --compress --wal-file-path %p --wal-file-name %f'".format(
+                        self.probackup_path, backup_dir, instance))
+            else:
+                node.append_conf(
                     "postgresql.auto.conf",
                     "archive_command = '{0} archive-push -B {1} --instance={2} --wal-file-path %p --wal-file-name %f'".format(
                         self.probackup_path, backup_dir, instance))

tests/option_test.py

@@ -26,9 +26,9 @@ def test_version_2(self):
         fname = self.id().split(".")[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out:
-            self.assertEqual(
-                self.run_pb(["--version"]),
-                version_out.read().decode("utf-8")
+            self.assertIn(
+                version_out.read().decode("utf-8"),
+                self.run_pb(["--version"])
             )
 
     # @unittest.skip("skip")
@@ -76,8 +76,8 @@ def test_options_4(self):
             self.assertEqual(1, 0, "Expecting Error because '-b' parameter is not specified.\n Output: {0} \n CMD: {1}".format(
                 repr(self.output), self.cmd))
         except ProbackupException as e:
-            self.assertEqual(e.message,
-                'ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)\n',
+            self.assertIn('ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)',
+                e.message,
                 '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
 
         # backup command failure with invalid backup mode option

tests/page.py

@@ -10,34 +10,34 @@
 
 class PageBackupTest(ProbackupTest, unittest.TestCase):
 
-    # @unittest.skip("skip")
-    # @unittest.expectedFailure
-    def test_page_check_archive_enabled(self):
-        """make node, take page backup without enabled archive, should result in error"""
-        self.maxDiff = None
-        fname = self.id().split('.')[3]
-        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
-        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
-            set_replication=True,
-            initdb_params=['--data-checksums'],
-            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'}
-            )
-
-        self.init_pb(backup_dir)
-        self.add_instance(backup_dir, 'node', node)
-        node.start()
-
-        try:
-            self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream'])
-            # we should die here because exception is what we expect to happen
-            self.assertEqual(1, 0, "Expecting Error because archive_mode disabled.\n Output: {0} \n CMD: {1}".format(
-                repr(self.output), self.cmd))
-        except ProbackupException as e:
-            self.assertEqual('ERROR: Archiving must be enabled for PAGE backup\n', e.message,
-                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
-
-        # Clean after yourself
-        self.del_test_dir(module_name, fname)
+#    # @unittest.skip("skip")
+#    # @unittest.expectedFailure
+#    def test_page_check_archive_enabled(self):
+#        """make node, take page backup without enabled archive, should result in error"""
+#        self.maxDiff = None
+#        fname = self.id().split('.')[3]
+#        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+#        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+#            set_replication=True,
+#            initdb_params=['--data-checksums'],
+#            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'}
+#            )
+#
+#        self.init_pb(backup_dir)
+#        self.add_instance(backup_dir, 'node', node)
+#        node.start()
+#
+#        try:
+#            self.backup_node(backup_dir, 'node', node, backup_type='page', options=['--stream'])
+#            # we should die here because exception is what we expect to happen
+#            self.assertEqual(1, 0, "Expecting Error because archive_mode disabled.\n Output: {0} \n CMD: {1}".format(
+#                repr(self.output), self.cmd))
+#        except ProbackupException as e:
+#            self.assertIn('ERROR: Archiving must be enabled for PAGE backup\n', e.message,
+#                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
+#
+#        # Clean after yourself
+#        self.del_test_dir(module_name, fname)
 
     # @unittest.skip("skip")
     def test_page_stream(self):
@@ -182,7 +182,7 @@ def test_page_multiple_segments(self):
         # GET LOGICAL CONTENT FROM NODE
         result = node.safe_psql("postgres", "select * from pgbench_accounts")
         # PAGE BACKUP
-        self.backup_node(backup_dir, 'node', node, backup_type='page', options=["-l", "--log-level=verbose"])
+        self.backup_node(backup_dir, 'node', node, backup_type='page', options=["--log-level-file=verbose"])
         # GET PHYSICAL CONTENT FROM NODE
         pgdata = self.pgdata_content(node.data_dir)
 

tests/pgpro589.py

@@ -10,45 +10,6 @@
 
 class ArchiveCheck(ProbackupTest, unittest.TestCase):
 
-    # @unittest.skip("skip")
-    # @unittest.expectedFailure
-    def test_archive_mode(self):
-        """
-        https://jira.postgrespro.ru/browse/PGPRO-589
-        make node without archive support, make backup which should fail
-        check ERROR text
-        """
-        fname = self.id().split('.')[3]
-        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
-            initdb_params=['--data-checksums'],
-            pg_options={'wal_level': 'replica'}
-            )
-        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
-        self.init_pb(backup_dir)
-        self.add_instance(backup_dir, 'node', node)
-        node.start()
-
-        node.pgbench_init(scale=5)
-        pgbench = node.pgbench(
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            options=["-c", "4", "-T", "10"]
-        )
-        pgbench.wait()
-        pgbench.stdout.close()
-
-        try:
-            self.backup_node(backup_dir, 'node', node, options=['--archive-timeout=10'])
-            # we should die here because exception is what we expect to happen
-            self.assertEqual(1, 0, "Expecting Error because of disabled archive_mode.\n Output: {0} \n CMD: {1}".format(
-                repr(self.output), self.cmd))
-        except ProbackupException as e:
-            self.assertEqual(e.message, 'ERROR: Archiving must be enabled for archive backup\n',
-                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
-
-        # Clean after yourself
-        self.del_test_dir(module_name, fname)
-
     def test_pgpro589(self):
         """
         https://jira.postgrespro.ru/browse/PGPRO-589

tests/ptrack.py

@@ -36,7 +36,7 @@ def test_ptrack_enable(self):
             self.assertEqual(1, 0, "Expecting Error because ptrack disabled.\n Output: {0} \n CMD: {1}".format(
                 repr(self.output), self.cmd))
         except ProbackupException as e:
-            self.assertEqual('ERROR: Ptrack is disabled\n', e.message,
+            self.assertIn('ERROR: Ptrack is disabled\n', e.message,
                 '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
 
         # Clean after yourself
@@ -391,15 +391,15 @@ def test_create_db(self):
         node.safe_psql("postgres",
             "create table t_heap as select i as id, md5(i::text) as text, md5(i::text)::tsvector as tsvector from generate_series(0,100) i")
         node.safe_psql("postgres", "SELECT * FROM t_heap")
-        self.backup_node(backup_dir, 'node', node, options=["--stream", "-l", "--log-level=verbose"])
+        self.backup_node(backup_dir, 'node', node, options=["--stream", "--log-level-file=verbose"])
         #sys.exit(1)
 
         # CREATE DATABASE DB1
         node.safe_psql("postgres", "create database db1")
         node.safe_psql("db1", "create table t_heap as select i as id, md5(i::text) as text, md5(i::text)::tsvector as tsvector from generate_series(0,100) i")
 
         # PTRACK BACKUP
-        backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream", "-l", "--log-level=verbose"])
+        backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream", "--log-level-file=verbose"])
         pgdata = self.pgdata_content(node.data_dir)
 
         # RESTORE
@@ -479,7 +479,7 @@ def test_alter_table_set_tablespace_ptrack(self):
         # sys.exit(1)
         # PTRACK BACKUP
         result =  node.safe_psql("postgres", "select * from t_heap")
-        self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream", "-l", "--log-level=verbose"])
+        self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream", "--log-level-file=verbose"])
         pgdata = self.pgdata_content(node.data_dir)
         #node.stop()
         #node.cleanup()
@@ -538,7 +538,7 @@ def test_alter_database_set_tablespace_ptrack(self):
         #sys.exit(1)
 
         # PTRACK BACKUP
-        self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream", "-l", '--log-level=verbose'])
+        self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream", '--log-level-file=verbose'])
         pgdata= self.pgdata_content(node.data_dir)
         node.stop()
 
@@ -655,7 +655,7 @@ def test_ptrack_alter_tablespace(self):
         result = node.safe_psql("postgres", "select * from t_heap")
 
         # FIRTS PTRACK BACKUP
-        self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream", "-l", "--log-level=verbose"])
+        self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream", "--log-level-file=verbose"])
 
         # GET PHYSICAL CONTENT FROM NODE
         pgdata = self.pgdata_content(node.data_dir)
@@ -691,7 +691,7 @@ def test_ptrack_alter_tablespace(self):
         node.safe_psql(
             "postgres", "alter table t_heap set tablespace pg_default")
         # SECOND PTRACK BACKUP
-        self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream", "-l", "--log-level=verbose"])
+        self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--stream", "--log-level-file=verbose"])
         pgdata = self.pgdata_content(node.data_dir)
 
         # Restore second ptrack backup and check table consistency
@@ -762,7 +762,7 @@ def test_ptrack_multiple_segments(self):
         # GET LOGICAL CONTENT FROM NODE
         result = node.safe_psql("postgres", "select * from pgbench_accounts")
         # FIRTS PTRACK BACKUP
-        self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["-l", "--log-level=verbose"])
+        self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=["--log-level-file=verbose"])
         # GET PHYSICAL CONTENT FROM NODE
         pgdata = self.pgdata_content(node.data_dir)