Skip to content

Commit 36e6440

Browse files
gsmolkgsmolk
committed
ptrack, validate, retention fixes
1 parent d470eff commit 36e6440

15 files changed

+376
-246
lines changed

‎tests/__init__.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
ptrack_vacuum_full, ptrack_vacuum_truncate
99

1010

11-
1211
def load_tests(loader, tests, pattern):
1312
suite = unittest.TestSuite()
1413
suite.addTests(loader.loadTestsFromModule(init_test))
@@ -17,8 +16,8 @@ def load_tests(loader, tests, pattern):
1716
suite.addTests(loader.loadTestsFromModule(backup_test))
1817
suite.addTests(loader.loadTestsFromModule(delete_test))
1918
suite.addTests(loader.loadTestsFromModule(restore_test))
20-
# suite.addTests(loader.loadTestsFromModule(validate_test))
21-
# suite.addTests(loader.loadTestsFromModule(retention_test))
19+
suite.addTests(loader.loadTestsFromModule(validate_test))
20+
suite.addTests(loader.loadTestsFromModule(retention_test))
2221
suite.addTests(loader.loadTestsFromModule(ptrack_clean))
2322
suite.addTests(loader.loadTestsFromModule(ptrack_cluster))
2423
suite.addTests(loader.loadTestsFromModule(ptrack_move_to_tablespace))

‎tests/option_test.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -133,11 +133,10 @@ def test_options_4(self):
133133
with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf:
134134
conf.write("retention-redundancy=1\n")
135135

136-
# TODO AFTER PGPRO-505
137-
# self.assertEqual(
138-
# self.retention_show(node, ["--redundancy", "2"]),
139-
# six.b("# retention policy\nREDUNDANCY=2\n")
140-
# )
136+
self.assertEqual(
137+
self.show_config(node)['retention-redundancy'],
138+
six.b('1')
139+
)
141140

142141
# User cannot send --system-identifier parameter via command line
143142
try:

‎tests/ptrack_clean.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ def test_ptrack_clean(self):
4444
# get path to heap and index files
4545
idx_ptrack[i]['path'] = self.get_fork_path(node, i)
4646
# get ptrack for every idx
47-
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_for_fork(
47+
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
4848
idx_ptrack[i]['path'], idx_ptrack[i]['size'])
4949
self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size'])
5050

@@ -61,7 +61,7 @@ def test_ptrack_clean(self):
6161
# update path to heap and index files in case they`ve changed
6262
idx_ptrack[i]['path'] = self.get_fork_path(node, i)
6363
# # get ptrack for every idx
64-
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_for_fork(
64+
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
6565
idx_ptrack[i]['path'], idx_ptrack[i]['size'])
6666
# check that ptrack bits are cleaned
6767
self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size'])
@@ -80,7 +80,7 @@ def test_ptrack_clean(self):
8080
# update path to heap and index files in case they`ve changed
8181
idx_ptrack[i]['path'] = self.get_fork_path(node, i)
8282
# # get ptrack for every idx
83-
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_for_fork(
83+
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
8484
idx_ptrack[i]['path'], idx_ptrack[i]['size'])
8585
# check that ptrack bits are cleaned
8686
self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size'])

‎tests/ptrack_cluster.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ def test_ptrack_cluster_btree(self):
6262
idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork(
6363
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
6464
# get ptrack for every idx
65-
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_for_fork(
65+
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
6666
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
6767

6868
# compare pages and check ptrack sanity
@@ -121,7 +121,7 @@ def test_ptrack_cluster_spgist(self):
121121
idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork(
122122
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
123123
# get ptrack for every idx
124-
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_for_fork(
124+
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
125125
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
126126

127127
# compare pages and check ptrack sanity
@@ -180,7 +180,7 @@ def test_ptrack_cluster_brin(self):
180180
idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork(
181181
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
182182
# get ptrack for every idx
183-
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_for_fork(
183+
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
184184
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
185185

186186
# compare pages and check ptrack sanity
@@ -239,7 +239,7 @@ def test_ptrack_cluster_gist(self):
239239
idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork(
240240
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
241241
# get ptrack for every idx
242-
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_for_fork(
242+
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
243243
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
244244

245245
# compare pages and check ptrack sanity
@@ -298,7 +298,7 @@ def test_ptrack_cluster_gin(self):
298298
idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork(
299299
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
300300
# get ptrack for every idx
301-
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_for_fork(
301+
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
302302
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
303303

304304
# compare pages and check ptrack sanity

‎tests/ptrack_helpers.py‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,8 @@ def make_bnode(self, base_dir=None, allows_streaming=False, options={}):
172172

173173
return node
174174

175+
# def print_started(self, fname):
176+
# print
175177

176178
def make_simple_node(self, base_dir=None, set_replication=False,
177179
set_archiving=False, initdb_params=[], pg_options={}):
@@ -234,23 +236,21 @@ def get_md5_per_page_for_fork(self, file, size):
234236
os.close(file)
235237
return md5_per_page
236238

237-
def get_ptrack_bits_per_for_fork(self, file, size):
239+
def get_ptrack_bits_per_page_for_fork(self, file, size):
240+
ptrack_bits_for_fork = []
238241
byte_size = os.path.getsize(file + '_ptrack')
239242
byte_size_minus_header = byte_size - 24
240243
file = os.open(file + '_ptrack', os.O_RDONLY)
241244
os.lseek(file, 24, 0)
242245
lot_of_bytes = os.read(file, byte_size_minus_header)
243-
ptrack_bits_per_for_fork = []
244246
for byte in lot_of_bytes:
245247
byte_inverted = bin(ord(byte))[2:].rjust(8, '0')[::-1]
246248
# byte_to_bits = (byte >> x) & 1 for x in range(7, -1, -1)
247249
for bit in byte_inverted:
248-
while len(ptrack_bits_per_for_fork) != size:
249-
ptrack_bits_per_for_fork.append(int(bit))
250-
# print 'Size: {}'.format(size)
251-
# print ptrack_bits_per_for_fork
250+
if len(ptrack_bits_for_fork) < size:
251+
ptrack_bits_for_fork.append(int(bit))
252252
os.close(file)
253-
return ptrack_bits_per_for_fork
253+
return ptrack_bits_for_fork
254254

255255
def check_ptrack_sanity(self, idx_dict):
256256
success = True
@@ -284,7 +284,7 @@ def check_ptrack_sanity(self, idx_dict):
284284
PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])
285285
print idx_dict
286286
if PageNum == 0 and idx_dict['type'] == 'spgist':
287-
print 'SPGIST is a special showflake, so don`t freat about losing ptrack for blknum 0'
287+
print 'SPGIST is a special snowflake, so don`t fret about losing ptrack for blknum 0'
288288
continue
289289
success = False
290290
else:
@@ -468,19 +468,26 @@ def delete_pb(self, node, id=None, options=[]):
468468
# print(cmd_list)
469469
return self.run_pb(cmd_list + options)
470470

471-
def retention_purge_pb(self, node, options=[]):
471+
def delete_expired(self, node, options=[]):
472472
cmd_list = [
473-
"retention", "purge",
473+
"delete", "--expired",
474474
"-B", self.backup_dir(node),
475475
]
476476
return self.run_pb(cmd_list + options)
477477

478-
def retention_show(self, node, options=[]):
478+
def show_config(self, node):
479+
out_dict = {}
479480
cmd_list = [
480-
"config",
481+
"show-config",
481482
"-B", self.backup_dir(node),
482483
]
483-
return self.run_pb(cmd_list + options)
484+
res = self.run_pb(cmd_list).splitlines()
485+
for line in res:
486+
if not line.startswith('#'):
487+
name, var = line.partition(" = ")[::2]
488+
out_dict[name] = var
489+
return out_dict
490+
484491

485492
def get_recovery_conf(self, node):
486493
out_dict = {}

‎tests/ptrack_move_to_tablespace.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ def test_ptrack_recovery(self):
4848
# get path to heap and index files
4949
idx_ptrack[i]['path'] = self.get_fork_path(node, i)
5050
# get ptrack for every idx
51-
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_for_fork(
51+
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
5252
idx_ptrack[i]['path'], idx_ptrack[i]['size'])
5353
# check that ptrack has correct bits after recovery
5454
self.check_ptrack_recovery(idx_ptrack[i])

‎tests/ptrack_recovery.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ def test_ptrack_recovery(self):
5050

5151
for i in idx_ptrack:
5252
# get ptrack for every idx
53-
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_for_fork(
53+
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
5454
idx_ptrack[i]['path'], idx_ptrack[i]['size'])
5555
# check that ptrack has correct bits after recovery
5656
self.check_ptrack_recovery(idx_ptrack[i])

‎tests/ptrack_vacuum.py‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,8 @@
11
import unittest
22
from sys import exit
33
from testgres import get_new_node, stop_all
4-
#import os
5-
from os import path, open, lseek, read, close, O_RDONLY
64
from .ptrack_helpers import ProbackupTest, idx_ptrack
75

8-
# res = node.execute('postgres', 'show fsync')
9-
# print res[0][0]
10-
# res = node.execute('postgres', 'show wal_level')
11-
# print res[0][0]
12-
# a = ProbackupTest
13-
# res = node.execute('postgres', 'select 1')`
14-
# self.assertEqual(len(res), 1)
15-
# self.assertEqual(res[0][0], 1)
16-
# node.stop()
17-
# a = self.backup_dir(node)
18-
196

207
class SimpleTest(ProbackupTest, unittest.TestCase):
218
def __init__(self, *args, **kwargs):
@@ -63,7 +50,7 @@ def test_ptrack_vacuum(self):
6350
self.init_pb(node)
6451
self.backup_pb(node, backup_type='full', options=['-j100', '--stream'])
6552
for i in idx_ptrack:
66-
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_for_fork(
53+
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
6754
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
6855
self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size'])
6956

@@ -81,7 +68,7 @@ def test_ptrack_vacuum(self):
8168
idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork(
8269
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
8370
# get ptrack for every idx
84-
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_for_fork(
71+
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
8572
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
8673

8774
# compare pages and check ptrack sanity

‎tests/ptrack_vacuum_bits_frozen.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ def test_ptrack_vacuum_bits_frozen(self):
5959
idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork(
6060
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
6161
# get ptrack for every idx
62-
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_for_fork(
62+
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
6363
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
6464

6565
# compare pages and check ptrack sanity

‎tests/ptrack_vacuum_bits_visibility.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ def test_ptrack_vacuum_bits_visibility(self):
5959
idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork(
6060
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
6161
# get ptrack for every idx
62-
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_for_fork(
62+
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
6363
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
6464

6565
# compare pages and check ptrack sanity

0 commit comments

Comments
 (0)