PGPRO-1918 tests: added test_locking_running_validate_2_specific_id()

Author: gsmolk

tests/helpers/ptrack_helpers.py

@@ -1383,14 +1383,26 @@ def continue_execution_until_exit(self):
             if line.startswith('*stopped,reason="breakpoint-hit"'):
                 continue
             if (
-                line.startswith('*stopped,reason="exited-normally"') or
+                line.startswith('*stopped,reason="exited"') or
                 line == '*stopped\n'
             ):
                 return
         raise GdbException(
             'Failed to continue execution until exit.\n'
         )
 
+    def continue_execution_until_error(self):
+        result = self._execute('continue', False)
+
+        for line in result:
+            if line.startswith('^error'):
+                return
+            if line.startswith('*stopped,reason="exited'):
+                return
+
+        raise GdbException(
+            'Failed to continue execution until error.\n')
+
     def continue_execution_until_break(self, ignore_count=0):
         if ignore_count > 0:
             result = self._execute(
@@ -1436,6 +1448,8 @@ def _execute(self, cmd, running=True):
                 print(repr(line))
             if line.startswith('^done') or line.startswith('*stopped'):
                 break
+            if line.startswith('^error'):
+                break
             if running and (line.startswith('*running') or line.startswith('^running')):
 #            if running and line.startswith('*running'):
                 break

tests/locking.py

@@ -11,7 +11,7 @@ class LockingTest(ProbackupTest, unittest.TestCase):
 
     # @unittest.skip("skip")
     # @unittest.expectedFailure
-    def test_locking_running_1(self):
+    def test_locking_running_validate_1(self):
         """
         make node, take full backup, stop it in the middle
         run validate, expect it to successfully executed,
@@ -46,18 +46,26 @@ def test_locking_running_1(self):
         self.assertEqual(
             'RUNNING', self.show_pb(backup_dir, 'node')[1]['status'])
 
-        self.validate_pb(backup_dir, options=['--log-level-file=VERBOSE'])
+        validate_output = self.validate_pb(
+            backup_dir, options=['--log-level-console=LOG'])
+
+        backup_id = self.show_pb(backup_dir, 'node')[1]['id']
+
+        self.assertIn(
+            "is using backup {0} and still is running".format(backup_id),
+            validate_output,
+            '\n Unexpected Validate Output: {0}\n'.format(repr(validate_output)))
 
         self.assertEqual(
             'OK', self.show_pb(backup_dir, 'node')[0]['status'])
 
         self.assertEqual(
-            'ERROR', self.show_pb(backup_dir, 'node')[1]['status'])
+            'RUNNING', self.show_pb(backup_dir, 'node')[1]['status'])
 
         # Clean after yourself
-        self.del_test_dir(module_name, fname)
+        # self.del_test_dir(module_name, fname)
 
-    def test_locking_running_2(self):
+    def test_locking_running_validate_2(self):
         """
         make node, take full backup, stop it in the middle,
         kill process so no cleanup is done - pid file is in place,
@@ -89,18 +97,105 @@ def test_locking_running_2(self):
             self.AssertTrue(False, 'Failed to hit breakpoint')
 
         gdb._execute('signal SIGKILL')
-        gdb.continue_execution_until_running()
+        gdb.continue_execution_until_error()
 
         self.assertEqual(
             'OK', self.show_pb(backup_dir, 'node')[0]['status'])
 
         self.assertEqual(
             'RUNNING', self.show_pb(backup_dir, 'node')[1]['status'])
 
+        backup_id = self.show_pb(backup_dir, 'node')[1]['id']
+
         try:
             self.validate_pb(backup_dir)
-        except:
-            pass
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because RUNNING backup is no longer active.\n "
+                "Output: {0} \n CMD: {1}".format(
+                    repr(self.output), self.cmd))
+        except ProbackupException as e:
+            self.assertTrue(
+                "which used backup {0} no longer exists".format(
+                    backup_id) in e.message and
+                "Backup {0} has status RUNNING, change it "
+                "to ERROR and skip validation".format(
+                    backup_id) in e.message and
+                "WARNING: Some backups are not valid" in
+                e.message,
+                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                    repr(e.message), self.cmd))
+
+        self.assertEqual(
+            'OK', self.show_pb(backup_dir, 'node')[0]['status'])
+
+        self.assertEqual(
+            'ERROR', self.show_pb(backup_dir, 'node')[1]['status'])
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
+
+    def test_locking_running_validate_2_specific_id(self):
+        """
+        make node, take full backup, stop it in the middle,
+        kill process so no cleanup is done - pid file is in place,
+        run validate on this specific backup,
+        expect it to not successfully executed,
+        RUNNING backup with pid file AND without active pid is legal,
+        but his status must be changed to ERROR and pid file is deleted
+        """
+        fname = self.id().split('.')[3]
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'),
+            initdb_params=['--data-checksums'],
+            pg_options={'wal_level': 'replica'})
+
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.slow_start()
+
+        self.backup_node(backup_dir, 'node', node)
+
+        gdb = self.backup_node(
+            backup_dir, 'node', node, gdb=True)
+
+        gdb.set_breakpoint('copy_file')
+        gdb.run_until_break()
+
+        if gdb.continue_execution_until_break(20) != 'breakpoint-hit':
+            self.AssertTrue(False, 'Failed to hit breakpoint')
+
+        gdb._execute('signal SIGKILL')
+        gdb.continue_execution_until_error()
+
+        self.assertEqual(
+            'OK', self.show_pb(backup_dir, 'node')[0]['status'])
+
+        self.assertEqual(
+            'RUNNING', self.show_pb(backup_dir, 'node')[1]['status'])
+
+        backup_id = self.show_pb(backup_dir, 'node')[1]['id']
+
+        try:
+            self.validate_pb(backup_dir, 'node', backup_id)
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because RUNNING backup is no longer active.\n "
+                "Output: {0} \n CMD: {1}".format(
+                    repr(self.output), self.cmd))
+        except ProbackupException as e:
+            self.assertTrue(
+                "which used backup {0} no longer exists".format(
+                    backup_id) in e.message and
+                "Backup {0} has status RUNNING, change it "
+                "to ERROR and skip validation".format(
+                    backup_id) in e.message and
+                "WARNING: Some backups are not valid" in
+                e.message,
+                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                    repr(e.message), self.cmd))
 
         self.assertEqual(
             'OK', self.show_pb(backup_dir, 'node')[0]['status'])
@@ -143,7 +238,7 @@ def test_locking_running_3(self):
             self.AssertTrue(False, 'Failed to hit breakpoint')
 
         gdb._execute('signal SIGKILL')
-        gdb.continue_execution_until_running()
+        gdb.continue_execution_until_error()
 
         self.assertEqual(
             'OK', self.show_pb(backup_dir, 'node')[0]['status'])
@@ -158,8 +253,20 @@ def test_locking_running_3(self):
 
         try:
             self.validate_pb(backup_dir)
-        except:
-            pass
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because RUNNING backup is no longer active.\n "
+                "Output: {0} \n CMD: {1}".format(
+                    repr(self.output), self.cmd))
+        except ProbackupException as e:
+            self.assertTrue(
+                "Backup {0} has status RUNNING, change it "
+                "to ERROR and skip validation".format(
+                    backup_id) in e.message and
+                "WARNING: Some backups are not valid" in
+                e.message,
+                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                    repr(e.message), self.cmd))
 
         self.assertEqual(
             'OK', self.show_pb(backup_dir, 'node')[0]['status'])