Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 037b7cf

Browse files
alubennikovaalubennikova
committed
Merge branch 'master' into issue_79
2 parents d92434f + 736fc41 commit 037b7cf

File tree

6 files changed

+197
-19
lines changed

6 files changed

+197
-19
lines changed

‎src/delete.c

+20-9
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
259259
{
260260

261261
bool redundancy_keep = false;
262+
time_t backup_time = 0;
262263
pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i);
263264

264265
/* check if backup`s FULL ancestor is in redundancy list */
@@ -280,10 +281,16 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
280281
cur_full_backup_num++;
281282
}
282283

283-
/* Check if backup in needed by retention policy
284-
* TODO: consider that ERROR backup most likely to have recovery_time == 0
284+
/* Invalid and running backups most likely to have recovery_time == 0,
285+
* so in this case use start_time instead.
285286
*/
286-
if ((days_threshold == 0 || (days_threshold > backup->recovery_time)) &&
287+
if (backup->recovery_time)
288+
backup_time = backup->recovery_time;
289+
else
290+
backup_time = backup->start_time;
291+
292+
/* Check if backup in needed by retention policy */
293+
if ((days_threshold == 0 || (days_threshold > backup_time)) &&
287294
(instance_config.retention_redundancy == 0 || !redundancy_keep))
288295
{
289296
/* This backup is not guarded by retention
@@ -622,6 +629,7 @@ do_retention_wal(void)
622629
XLogRecPtr oldest_lsn = InvalidXLogRecPtr;
623630
TimeLineID oldest_tli = 0;
624631
bool backup_list_is_empty = false;
632+
int i;
625633

626634
/* Get list of backups. */
627635
backup_list = catalog_get_backup_list(INVALID_BACKUP_ID);
@@ -630,14 +638,17 @@ do_retention_wal(void)
630638
backup_list_is_empty = true;
631639

632640
/* Save LSN and Timeline to remove unnecessary WAL segments */
633-
if (!backup_list_is_empty)
641+
for (i = (int) parray_num(backup_list) - 1; i >= 0; i--)
634642
{
635-
pgBackup *backup = NULL;
636-
/* Get LSN and TLI of oldest alive backup */
637-
backup = (pgBackup *) parray_get(backup_list, parray_num(backup_list) -1);
643+
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
638644

639-
oldest_tli = backup->tli;
640-
oldest_lsn = backup->start_lsn;
645+
/* Get LSN and TLI of the oldest backup with valid start_lsn and tli */
646+
if (backup->tli > 0 && !XLogRecPtrIsInvalid(backup->start_lsn))
647+
{
648+
oldest_tli = backup->tli;
649+
oldest_lsn = backup->start_lsn;
650+
break;
651+
}
641652
}
642653

643654
/* Be paranoid */

‎src/pg_probackup.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -663,7 +663,7 @@ main(int argc, char *argv[])
663663
recovery_target_options,
664664
restore_params);
665665
case VALIDATE_CMD:
666-
if (current.backup_id == 0 && target_time == 0 && target_xid == 0)
666+
if (current.backup_id == 0 && target_time == 0 && target_xid == 0 && !target_lsn)
667667
return do_validate_all();
668668
else
669669
return do_restore_or_validate(current.backup_id,

‎src/restore.c

+14-8
Original file line numberDiff line numberDiff line change
@@ -143,10 +143,16 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
143143
/*
144144
* [PGPRO-1164] If BACKUP_ID is not provided for restore command,
145145
* we must find the first valid(!) backup.
146+
147+
* If target_backup_id is not provided, we can be sure that
148+
* PITR for restore or validate is requested.
149+
* So we can assume that user is more interested in recovery to specific point
150+
* in time and NOT interested in revalidation of invalid backups.
151+
* So based on that assumptions we should choose only OK and DONE backups
152+
* as candidates for validate and restore.
146153
*/
147154

148-
if (params->is_restore &&
149-
target_backup_id == INVALID_BACKUP_ID &&
155+
if (target_backup_id == INVALID_BACKUP_ID &&
150156
(current_backup->status != BACKUP_STATUS_OK &&
151157
current_backup->status != BACKUP_STATUS_DONE))
152158
{
@@ -1079,7 +1085,7 @@ parseRecoveryTargetOptions(const char *target_time,
10791085
if (parse_time(target_time, &dummy_time, false))
10801086
rt->target_time = dummy_time;
10811087
else
1082-
elog(ERROR, "Invalid value for --recovery-target-time option %s",
1088+
elog(ERROR, "Invalid value for '--recovery-target-time' option %s",
10831089
target_time);
10841090
}
10851091

@@ -1097,7 +1103,7 @@ parseRecoveryTargetOptions(const char *target_time,
10971103
#endif
10981104
rt->target_xid = dummy_xid;
10991105
else
1100-
elog(ERROR, "Invalid value for --recovery-target-xid option %s",
1106+
elog(ERROR, "Invalid value for '--recovery-target-xid' option %s",
11011107
target_xid);
11021108
}
11031109

@@ -1110,7 +1116,7 @@ parseRecoveryTargetOptions(const char *target_time,
11101116
if (parse_lsn(target_lsn, &dummy_lsn))
11111117
rt->target_lsn = dummy_lsn;
11121118
else
1113-
elog(ERROR, "Invalid value of --recovery-target-lsn option %s",
1119+
elog(ERROR, "Invalid value of '--recovery-target-lsn' option %s",
11141120
target_lsn);
11151121
}
11161122

@@ -1120,7 +1126,7 @@ parseRecoveryTargetOptions(const char *target_time,
11201126
if (parse_bool(target_inclusive, &dummy_bool))
11211127
rt->target_inclusive = dummy_bool;
11221128
else
1123-
elog(ERROR, "Invalid value for --recovery-target-inclusive option %s",
1129+
elog(ERROR, "Invalid value for '--recovery-target-inclusive' option %s",
11241130
target_inclusive);
11251131
}
11261132

@@ -1129,7 +1135,7 @@ parseRecoveryTargetOptions(const char *target_time,
11291135
{
11301136
if ((strcmp(target_stop, "immediate") != 0)
11311137
&& (strcmp(target_stop, "latest") != 0))
1132-
elog(ERROR, "Invalid value for --recovery-target option %s",
1138+
elog(ERROR, "Invalid value for '--recovery-target' option %s",
11331139
target_stop);
11341140

11351141
recovery_target_specified++;
@@ -1147,7 +1153,7 @@ parseRecoveryTargetOptions(const char *target_time,
11471153
if ((strcmp(target_action, "pause") != 0)
11481154
&& (strcmp(target_action, "promote") != 0)
11491155
&& (strcmp(target_action, "shutdown") != 0))
1150-
elog(ERROR, "Invalid value for --recovery-target-action option %s",
1156+
elog(ERROR, "Invalid value for '--recovery-target-action' option %s",
11511157
target_action);
11521158

11531159
rt->target_action = target_action;

‎tests/backup.py

+6
Original file line numberDiff line numberDiff line change
@@ -1437,6 +1437,9 @@ def test_drop_table(self):
14371437
# @unittest.skip("skip")
14381438
def test_basic_missing_file_permissions(self):
14391439
""""""
1440+
if os.name == 'nt':
1441+
return unittest.skip('Skipped because it is POSIX only test')
1442+
14401443
fname = self.id().split('.')[3]
14411444
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
14421445
node = self.make_simple_node(
@@ -1481,6 +1484,9 @@ def test_basic_missing_file_permissions(self):
14811484
# @unittest.skip("skip")
14821485
def test_basic_missing_dir_permissions(self):
14831486
""""""
1487+
if os.name == 'nt':
1488+
return unittest.skip('Skipped because it is POSIX only test')
1489+
14841490
fname = self.id().split('.')[3]
14851491
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
14861492
node = self.make_simple_node(

‎tests/retention.py

+43-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import os
22
import unittest
33
from datetime import datetime, timedelta
4-
from .helpers.ptrack_helpers import ProbackupTest
4+
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
55
from time import sleep
66

77

@@ -1459,3 +1459,45 @@ def test_retention_redundancy_overlapping_chains(self):
14591459

14601460
# Clean after yourself
14611461
self.del_test_dir(module_name, fname)
1462+
1463+
def test_wal_purge_victim(self):
1464+
"""
1465+
https://github.com/postgrespro/pg_probackup/issues/103
1466+
"""
1467+
fname = self.id().split('.')[3]
1468+
node = self.make_simple_node(
1469+
base_dir=os.path.join(module_name, fname, 'node'),
1470+
initdb_params=['--data-checksums'])
1471+
1472+
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
1473+
self.init_pb(backup_dir)
1474+
self.add_instance(backup_dir, 'node', node)
1475+
self.set_archiving(backup_dir, 'node', node)
1476+
node.slow_start()
1477+
1478+
# Make ERROR incremental backup
1479+
try:
1480+
self.backup_node(backup_dir, 'node', node, backup_type='page')
1481+
# we should die here because exception is what we expect to happen
1482+
self.assertEqual(
1483+
1, 0,
1484+
"Expecting Error because page backup should not be possible "
1485+
"without valid full backup.\n Output: {0} \n CMD: {1}".format(
1486+
repr(self.output), self.cmd))
1487+
except ProbackupException as e:
1488+
self.assertIn(
1489+
"ERROR: Valid backup on current timeline 1 is not found. "
1490+
"Create new FULL backup before an incremental one.",
1491+
e.message,
1492+
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
1493+
repr(e.message), self.cmd))
1494+
1495+
page_id = self.show_pb(backup_dir, 'node')[0]['id']
1496+
1497+
sleep(1)
1498+
1499+
# Make FULL backup
1500+
self.backup_node(backup_dir, 'node', node, options=['--delete-wal'])
1501+
1502+
# Clean after yourself
1503+
self.del_test_dir(module_name, fname)

‎tests/validate.py

+113
Original file line numberDiff line numberDiff line change
@@ -3443,6 +3443,119 @@ def test_validate_corrupt_tablespace_map(self):
34433443
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
34443444
repr(e.message), self.cmd))
34453445

3446+
# @unittest.expectedFailure
3447+
# @unittest.skip("skip")
3448+
def test_validate_target_lsn(self):
3449+
"""
3450+
Check validation to specific LSN
3451+
"""
3452+
fname = self.id().split('.')[3]
3453+
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
3454+
node = self.make_simple_node(
3455+
base_dir=os.path.join(module_name, fname, 'node'),
3456+
set_replication=True,
3457+
initdb_params=['--data-checksums'])
3458+
3459+
self.init_pb(backup_dir)
3460+
self.add_instance(backup_dir, 'node', node)
3461+
self.set_archiving(backup_dir, 'node', node)
3462+
node.slow_start()
3463+
3464+
# FULL backup
3465+
self.backup_node(backup_dir, 'node', node)
3466+
3467+
node.safe_psql(
3468+
"postgres",
3469+
"create table t_heap as select 1 as id, md5(i::text) as text, "
3470+
"md5(repeat(i::text,10))::tsvector as tsvector "
3471+
"from generate_series(0,10000) i")
3472+
3473+
node_restored = self.make_simple_node(
3474+
base_dir=os.path.join(module_name, fname, 'node_restored'))
3475+
node_restored.cleanup()
3476+
3477+
self.restore_node(backup_dir, 'node', node_restored)
3478+
3479+
node_restored.append_conf(
3480+
"postgresql.auto.conf", "port = {0}".format(node_restored.port))
3481+
3482+
node_restored.slow_start()
3483+
3484+
self.switch_wal_segment(node)
3485+
3486+
backup_id = self.backup_node(
3487+
backup_dir, 'node', node_restored,
3488+
data_dir=node_restored.data_dir)
3489+
3490+
target_lsn = self.show_pb(backup_dir, 'node')[1]['stop-lsn']
3491+
3492+
self.delete_pb(backup_dir, 'node', backup_id)
3493+
3494+
self.validate_pb(
3495+
backup_dir, 'node',
3496+
options=[
3497+
'--recovery-target-timeline=2',
3498+
'--recovery-target-lsn={0}'.format(target_lsn)])
3499+
3500+
# @unittest.expectedFailure
3501+
# @unittest.skip("skip")
3502+
def test_recovery_target_backup_victim(self):
3503+
"""
3504+
Check that for validation to recovery target
3505+
probackup chooses valid backup
3506+
https://github.com/postgrespro/pg_probackup/issues/104
3507+
"""
3508+
fname = self.id().split('.')[3]
3509+
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
3510+
node = self.make_simple_node(
3511+
base_dir=os.path.join(module_name, fname, 'node'),
3512+
set_replication=True,
3513+
initdb_params=['--data-checksums'])
3514+
3515+
self.init_pb(backup_dir)
3516+
self.add_instance(backup_dir, 'node', node)
3517+
self.set_archiving(backup_dir, 'node', node)
3518+
node.slow_start()
3519+
3520+
# FULL backup
3521+
self.backup_node(backup_dir, 'node', node)
3522+
3523+
node.safe_psql(
3524+
"postgres",
3525+
"create table t_heap as select 1 as id, md5(i::text) as text, "
3526+
"md5(repeat(i::text,10))::tsvector as tsvector "
3527+
"from generate_series(0,10000) i")
3528+
3529+
target_time = node.safe_psql(
3530+
"postgres",
3531+
"select now()").rstrip()
3532+
3533+
node.safe_psql(
3534+
"postgres",
3535+
"create table t_heap1 as select 1 as id, md5(i::text) as text, "
3536+
"md5(repeat(i::text,10))::tsvector as tsvector "
3537+
"from generate_series(0,100) i")
3538+
3539+
gdb = self.backup_node(backup_dir, 'node', node, gdb=True)
3540+
3541+
gdb.set_breakpoint('pg_stop_backup')
3542+
gdb.run_until_break()
3543+
gdb.remove_all_breakpoints()
3544+
gdb._execute('signal SIGINT')
3545+
gdb.continue_execution_until_error()
3546+
3547+
backup_id = self.show_pb(backup_dir, 'node')[1]['id']
3548+
3549+
self.assertEqual(
3550+
'ERROR',
3551+
self.show_pb(backup_dir, 'node', backup_id)['status'],
3552+
'Backup STATUS should be "ERROR"')
3553+
3554+
self.validate_pb(
3555+
backup_dir, 'node',
3556+
options=['--recovery-target-time={0}'.format(target_time)])
3557+
3558+
34463559
# validate empty backup list
34473560
# page from future during validate
34483561
# page from future during backup

0 commit comments

Comments
 (0)