Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit dff482a

Browse files
gsmolkgsmolk
committed
tests: ptrack tests major revamp
1 parent c9823c0 commit dff482a

12 files changed

+557
-146
lines changed

‎tests/helpers/ptrack_helpers.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -511,10 +511,10 @@ def check_ptrack_sanity(self, idx_dict):
511511
idx_dict['ptrack'][PageNum]
512512
)
513513
)
514-
515-
self.assertTrue(
516-
success, 'Ptrack has failed to register changes in data files'
517-
)
514+
return success
515+
# self.assertTrue(
516+
# success, 'Ptrack has failed to register changes in data files'
517+
# )
518518

519519
def check_ptrack_recovery(self, idx_dict):
520520
size = idx_dict['size']

‎tests/ptrack.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -1495,7 +1495,11 @@ def test_ptrack_multiple_segments(self):
14951495
node,
14961496
idx_ptrack['path']
14971497
)
1498-
self.check_ptrack_sanity(idx_ptrack)
1498+
1499+
if not self.check_ptrack_sanity(idx_ptrack):
1500+
self.assertTrue(
1501+
False, 'Ptrack has failed to register changes in data files'
1502+
)
14991503

15001504
# GET LOGICAL CONTENT FROM NODE
15011505
result = node.safe_psql("postgres", "select * from pgbench_accounts")

‎tests/ptrack_clean.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ def test_ptrack_clean(self):
3737
"as select i as id, nextval('t_seq') as t_seq, "
3838
"md5(i::text) as text, "
3939
"md5(repeat(i::text,10))::tsvector as tsvector "
40-
"from generate_series(0,256) i")
40+
"from generate_series(0,2560) i")
4141
for i in idx_ptrack:
4242
if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
4343
node.safe_psql(
@@ -154,7 +154,7 @@ def test_ptrack_clean_replica(self):
154154
"create sequence t_seq; create table t_heap as select i as id, "
155155
"nextval('t_seq') as t_seq, md5(i::text) as text, "
156156
"md5(repeat(i::text,10))::tsvector as tsvector "
157-
"from generate_series(0,256) i")
157+
"from generate_series(0,2560) i")
158158
for i in idx_ptrack:
159159
if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
160160
master.safe_psql(

‎tests/ptrack_cluster.py

+146-37
Large diffs are not rendered by default.

‎tests/ptrack_move_to_tablespace.py

+24-8
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
1212
# @unittest.expectedFailure
1313
def test_ptrack_recovery(self):
1414
fname = self.id().split('.')[3]
15-
node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
15+
node = self.make_simple_node(
16+
base_dir="{0}/{1}/node".format(module_name, fname),
1617
set_replication=True,
1718
initdb_params=['--data-checksums'],
18-
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
19+
pg_options={
20+
'ptrack_enable': 'on',
21+
'wal_level': 'replica',
22+
'max_wal_senders': '2'})
23+
1924
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
2025
self.init_pb(backup_dir)
2126
self.add_instance(backup_dir, 'node', node)
@@ -24,19 +29,30 @@ def test_ptrack_recovery(self):
2429
self.create_tblspace_in_node(node, 'somedata')
2530

2631
# Create table and indexes
27-
node.safe_psql("postgres",
28-
"create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text,md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
32+
node.safe_psql(
33+
"postgres",
34+
"create sequence t_seq; create table t_heap as select i as id, "
35+
"md5(i::text) as text,md5(repeat(i::text,10))::tsvector as "
36+
"tsvector from generate_series(0,2560) i")
37+
2938
for i in idx_ptrack:
3039
if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
31-
node.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format(
32-
i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
40+
node.safe_psql(
41+
"postgres",
42+
"create index {0} on {1} using {2}({3})".format(
43+
i, idx_ptrack[i]['relation'],
44+
idx_ptrack[i]['type'], idx_ptrack[i]['column']))
3345

3446
# Move table and indexes and make checkpoint
3547
for i in idx_ptrack:
3648
if idx_ptrack[i]['type'] == 'heap':
37-
node.safe_psql('postgres', 'alter table {0} set tablespace somedata;'.format(i))
49+
node.safe_psql(
50+
'postgres',
51+
'alter table {0} set tablespace somedata;'.format(i))
3852
if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
39-
node.safe_psql('postgres', 'alter index {0} set tablespace somedata'.format(i))
53+
node.safe_psql(
54+
'postgres',
55+
'alter index {0} set tablespace somedata'.format(i))
4056
node.safe_psql('postgres', 'checkpoint')
4157

4258
# Check ptrack files

‎tests/ptrack_recovery.py

+19-6
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
1313
# @unittest.expectedFailure
1414
def test_ptrack_recovery(self):
1515
fname = self.id().split('.')[3]
16-
node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
16+
node = self.make_simple_node(
17+
base_dir="{0}/{1}/node".format(module_name, fname),
1718
set_replication=True,
1819
initdb_params=['--data-checksums'],
19-
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
20+
pg_options={
21+
'ptrack_enable': 'on',
22+
'wal_level': 'replica',
23+
'max_wal_senders': '2'})
24+
2025
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
2126
self.init_pb(backup_dir)
2227
self.add_instance(backup_dir, 'node', node)
@@ -25,13 +30,21 @@ def test_ptrack_recovery(self):
2530
self.create_tblspace_in_node(node, 'somedata')
2631

2732
# Create table
28-
node.safe_psql("postgres",
29-
"create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text,md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
33+
node.safe_psql(
34+
"postgres",
35+
"create sequence t_seq; create table t_heap tablespace somedata "
36+
"as select i as id, md5(i::text) as text, "
37+
"md5(repeat(i::text,10))::tsvector as tsvector "
38+
"from generate_series(0,2560) i")
39+
3040
# Create indexes
3141
for i in idx_ptrack:
3242
if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
33-
node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format(
34-
i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
43+
node.safe_psql(
44+
"postgres", "create index {0} on {1} using {2}({3}) "
45+
"tablespace somedata".format(
46+
i, idx_ptrack[i]['relation'],
47+
idx_ptrack[i]['type'], idx_ptrack[i]['column']))
3548

3649
# get size of heap and indexes. size calculated in pages
3750
idx_ptrack[i]['size'] = int(self.get_fork_size(node, i))

‎tests/ptrack_truncate.py

+69-15
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,15 @@ class SimpleTest(ProbackupTest, unittest.TestCase):
1212
# @unittest.expectedFailure
1313
def test_ptrack_truncate(self):
1414
fname = self.id().split('.')[3]
15-
node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
15+
node = self.make_simple_node(
16+
base_dir="{0}/{1}/node".format(module_name, fname),
1617
set_replication=True,
1718
initdb_params=['--data-checksums'],
18-
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
19+
pg_options={
20+
'ptrack_enable': 'on',
21+
'wal_level': 'replica',
22+
'max_wal_senders': '2'})
23+
1924
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
2025
self.init_pb(backup_dir)
2126
self.add_instance(backup_dir, 'node', node)
@@ -26,11 +31,19 @@ def test_ptrack_truncate(self):
2631
# Create table and indexes
2732
node.safe_psql(
2833
"postgres",
29-
"create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
34+
"create sequence t_seq; create table t_heap tablespace somedata "
35+
"as select i as id, md5(i::text) as text, "
36+
"md5(repeat(i::text,10))::tsvector as tsvector "
37+
"from generate_series(0,2560) i")
38+
3039
for i in idx_ptrack:
3140
if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
32-
node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format(
33-
i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
41+
node.safe_psql(
42+
"postgres",
43+
"create index {0} on {1} using {2}({3}) "
44+
"tablespace somedata".format(
45+
i, idx_ptrack[i]['relation'],
46+
idx_ptrack[i]['type'], idx_ptrack[i]['column']))
3447

3548
node.safe_psql('postgres', 'truncate t_heap')
3649
node.safe_psql('postgres', 'checkpoint')
@@ -45,7 +58,8 @@ def test_ptrack_truncate(self):
4558
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
4659

4760
# Make full backup to clean every ptrack
48-
self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream'])
61+
self.backup_node(
62+
backup_dir, 'node', node, options=['-j10', '--stream'])
4963
for i in idx_ptrack:
5064
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
5165
node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']])
@@ -57,18 +71,25 @@ def test_ptrack_truncate(self):
5771
# @unittest.skip("skip")
5872
def test_ptrack_truncate_replica(self):
5973
fname = self.id().split('.')[3]
60-
master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname),
74+
master = self.make_simple_node(
75+
base_dir="{0}/{1}/master".format(module_name, fname),
6176
set_replication=True,
6277
initdb_params=['--data-checksums'],
63-
pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'})
78+
pg_options={
79+
'ptrack_enable': 'on',
80+
'wal_level': 'replica',
81+
'max_wal_senders': '2',
82+
'checkpoint_timeout': '30'})
83+
6484
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
6585
self.init_pb(backup_dir)
6686
self.add_instance(backup_dir, 'master', master)
6787
master.start()
6888

6989
self.backup_node(backup_dir, 'master', master, options=['--stream'])
7090

71-
replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname))
91+
replica = self.make_simple_node(
92+
base_dir="{0}/{1}/replica".format(module_name, fname))
7293
replica.cleanup()
7394

7495
self.restore_node(backup_dir, 'master', replica)
@@ -81,15 +102,30 @@ def test_ptrack_truncate_replica(self):
81102
# Create table and indexes
82103
master.safe_psql(
83104
"postgres",
84-
"create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
105+
"create sequence t_seq; create table t_heap tablespace somedata "
106+
"as select i as id, md5(i::text) as text, "
107+
"md5(repeat(i::text,10))::tsvector as tsvector "
108+
"from generate_series(0,2560) i")
85109
for i in idx_ptrack:
86110
if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq':
87-
master.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format(
88-
i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column']))
111+
master.safe_psql(
112+
"postgres", "create index {0} on {1} using {2}({3}) "
113+
"tablespace somedata".format(
114+
i, idx_ptrack[i]['relation'],
115+
idx_ptrack[i]['type'], idx_ptrack[i]['column']))
89116

90117
replica.safe_psql('postgres', 'truncate t_heap')
91118
replica.safe_psql('postgres', 'checkpoint')
92119

120+
# Sync master and replica
121+
lsn = master.safe_psql(
122+
'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
123+
replica.poll_query_until(
124+
"postgres",
125+
"SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
126+
lsn))
127+
replica.safe_psql('postgres', 'checkpoint')
128+
93129
for i in idx_ptrack:
94130
# get fork size and calculate it in pages
95131
idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i)
@@ -100,7 +136,8 @@ def test_ptrack_truncate_replica(self):
100136
idx_ptrack[i]['path'], idx_ptrack[i]['old_size'])
101137

102138
# Make full backup to clean every ptrack
103-
self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream'])
139+
self.backup_node(
140+
backup_dir, 'replica', replica, options=['-j10', '--stream'])
104141
for i in idx_ptrack:
105142
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
106143
replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']])
@@ -111,6 +148,17 @@ def test_ptrack_truncate_replica(self):
111148
master.safe_psql('postgres', 'vacuum t_heap')
112149
master.safe_psql('postgres', 'checkpoint')
113150

151+
# Sync master and replica
152+
lsn = master.safe_psql(
153+
'postgres', 'SELECT pg_catalog.pg_current_wal_lsn()').rstrip()
154+
replica.poll_query_until(
155+
"postgres",
156+
"SELECT '{0}'::pg_lsn <= pg_last_wal_replay_lsn()".format(
157+
lsn))
158+
replica.safe_psql('postgres', 'checkpoint')
159+
160+
# CHECK PTRACK SANITY
161+
success = True
114162
for i in idx_ptrack:
115163
# get new size of heap and indexes and calculate it in pages
116164
idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i)
@@ -121,10 +169,16 @@ def test_ptrack_truncate_replica(self):
121169
idx_ptrack[i]['path'], idx_ptrack[i]['new_size'])
122170
# get ptrack for every idx
123171
idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork(
124-
replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
172+
replica, idx_ptrack[i]['path'],
173+
[idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']])
125174

126175
# compare pages and check ptrack sanity
127-
self.check_ptrack_sanity(idx_ptrack[i])
176+
if not self.check_ptrack_sanity(idx_ptrack[i]):
177+
success = False
178+
179+
self.assertTrue(
180+
success, 'Ptrack has failed to register changes in data files'
181+
)
128182

129183
# Clean after yourself
130184
self.del_test_dir(module_name, fname)

0 commit comments

Comments
 (0)