Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 4a94fda

Browse files
gsmolkgsmolk
committed
[Issue #143] Multi-timeline incremental chain
1 parent a196073 commit 4a94fda

File tree

11 files changed

+1107
-165
lines changed

11 files changed

+1107
-165
lines changed

‎doc/pgprobackup.xml

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -426,14 +426,6 @@ doc/src/sgml/pgprobackup.sgml
426426
or <application>libc</application>/<application>libicu</application> versions.
427427
</para>
428428
</listitem>
429-
<listitem>
430-
<para>
431-
All backups in the incremental chain must belong to the same
432-
timeline. For example, if you have taken incremental backups on a
433-
standby server that gets promoted, you have to take another FULL
434-
backup.
435-
</para>
436-
</listitem>
437429
</itemizedlist>
438430
</para>
439431
</refsect2>
@@ -753,9 +745,10 @@ ALTER ROLE backup WITH REPLICATION;
753745
<title>Setting up Continuous WAL Archiving</title>
754746
<para>
755747
Making backups in PAGE backup mode, performing
756-
<link linkend="pbk-performing-point-in-time-pitr-recovery">PITR</link>
757-
and making backups with
758-
<link linkend="pbk-archive-mode">ARCHIVE</link> WAL delivery mode
748+
<link linkend="pbk-performing-point-in-time-pitr-recovery">PITR</link>,
749+
making backups with
750+
<link linkend="pbk-archive-mode">ARCHIVE</link> WAL delivery mode and
751+
running incremental backup after timeline switch
759752
require
760753
<ulink url="https://postgrespro.com/docs/postgresql/current/continuous-archiving.html">continuous
761754
WAL archiving</ulink> to be enabled. To set up continuous

‎src/backup.c

Lines changed: 63 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
153153
PGconn *master_conn = NULL;
154154
PGconn *pg_startbackup_conn = NULL;
155155

156+
/* used for multitimeline incremental backup */
157+
parray *tli_list = NULL;
158+
159+
156160
/* for fancy reporting */
157161
time_t start_time, end_time;
158162
char pretty_time[20];
@@ -181,17 +185,43 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
181185
current.backup_mode == BACKUP_MODE_DIFF_PTRACK ||
182186
current.backup_mode == BACKUP_MODE_DIFF_DELTA)
183187
{
184-
char prev_backup_filelist_path[MAXPGPATH];
185-
186188
/* get list of backups already taken */
187189
backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID);
188190

189191
prev_backup = catalog_get_last_data_backup(backup_list, current.tli, current.start_time);
190192
if (prev_backup == NULL)
191-
elog(ERROR, "Valid backup on current timeline %X is not found. "
192-
"Create new FULL backup before an incremental one.",
193+
{
194+
/* try to setup multi-timeline backup chain */
195+
elog(WARNING, "Valid backup on current timeline %u is not found, "
196+
"try to look up on previous timelines",
193197
current.tli);
194198

199+
tli_list = catalog_get_timelines(&instance_config);
200+
201+
if (parray_num(tli_list) == 0)
202+
elog(WARNING, "Cannot find valid backup on previous timelines, "
203+
"WAL archive is not available");
204+
else
205+
{
206+
prev_backup = get_multi_timeline_parent(backup_list, tli_list, current.tli,
207+
current.start_time, &instance_config);
208+
209+
if (prev_backup == NULL)
210+
elog(WARNING, "Cannot find valid backup on previous timelines");
211+
}
212+
213+
/* failed to find suitable parent, error out */
214+
if (!prev_backup)
215+
elog(ERROR, "Create new full backup before an incremental one");
216+
}
217+
}
218+
219+
if (prev_backup)
220+
{
221+
char prev_backup_filelist_path[MAXPGPATH];
222+
223+
elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time));
224+
195225
join_path_components(prev_backup_filelist_path, prev_backup->root_dir,
196226
DATABASE_FILE_LIST);
197227
/* Files of previous backup needed by DELTA backup */
@@ -378,8 +408,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
378408
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
379409
current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
380410
{
381-
elog(INFO, "Compiling pagemap of changed blocks");
411+
bool pagemap_isok = true;
412+
382413
time(&start_time);
414+
elog(INFO, "Extracting pagemap of changed blocks");
383415

384416
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE)
385417
{
@@ -388,8 +420,9 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
388420
* reading WAL segments present in archives up to the point
389421
* where this backup has started.
390422
*/
391-
extractPageMap(arclog_path, current.tli, instance_config.xlog_seg_size,
392-
prev_backup->start_lsn, current.start_lsn);
423+
pagemap_isok = extractPageMap(arclog_path, instance_config.xlog_seg_size,
424+
prev_backup->start_lsn, prev_backup->tli,
425+
current.start_lsn, current.tli, tli_list);
393426
}
394427
else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
395428
{
@@ -407,8 +440,14 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
407440
}
408441

409442
time(&end_time);
410-
elog(INFO, "Pagemap compiled, time elapsed %.0f sec",
411-
difftime(end_time, start_time));
443+
444+
/* TODO: add ms precision */
445+
if (pagemap_isok)
446+
elog(INFO, "Pagemap successfully extracted, time elapsed %.0f sec",
447+
difftime(end_time, start_time));
448+
else
449+
elog(ERROR, "Pagemap extraction failed, time elasped: %.0f sec",
450+
difftime(end_time, start_time));
412451
}
413452

414453
/*
@@ -667,6 +706,15 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
667706
elog(INFO, "Backup files are synced, time elapsed: %s", pretty_time);
668707
}
669708

709+
/* be paranoid about instance been from the past */
710+
if (current.backup_mode != BACKUP_MODE_FULL &&
711+
current.stop_lsn < prev_backup->stop_lsn)
712+
elog(ERROR, "Current backup STOP LSN %X/%X is lower than STOP LSN %X/%X of previous backup %s. "
713+
"It may indicate that we are trying to backup PostgreSQL instance from the past.",
714+
(uint32) (current.stop_lsn >> 32), (uint32) (current.stop_lsn),
715+
(uint32) (prev_backup->stop_lsn >> 32), (uint32) (prev_backup->stop_lsn),
716+
base36enc(prev_backup->stop_lsn));
717+
670718
/* clean external directories list */
671719
if (external_dirs)
672720
free_dir_list(external_dirs);
@@ -678,6 +726,12 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
678726
parray_free(backup_list);
679727
}
680728

729+
if (tli_list)
730+
{
731+
parray_walk(tli_list, timelineInfoFree);
732+
parray_free(tli_list);
733+
}
734+
681735
parray_walk(backup_files_list, pgFileFree);
682736
parray_free(backup_files_list);
683737
backup_files_list = NULL;

‎src/catalog.c

Lines changed: 161 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,24 @@ timelineInfoNew(TimeLineID tli)
4242
return tlinfo;
4343
}
4444

45+
/* free timelineInfo object */
46+
void
47+
timelineInfoFree(void *tliInfo)
48+
{
49+
timelineInfo *tli = (timelineInfo *) tliInfo;
50+
51+
parray_walk(tli->xlog_filelist, pgFileFree);
52+
parray_free(tli->xlog_filelist);
53+
54+
if (tli->backups)
55+
{
56+
parray_walk(tli->backups, pgBackupFree);
57+
parray_free(tli->backups);
58+
}
59+
60+
pfree(tliInfo);
61+
}
62+
4563
/* Iterate over locked backups and delete locks files */
4664
static void
4765
unlink_lock_atexit(void)
@@ -621,11 +639,7 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current
621639
* anomalies.
622640
*/
623641
if (is_parent(full_backup->start_time, backup, true))
624-
{
625-
elog(INFO, "Parent backup: %s",
626-
base36enc(backup->start_time));
627642
return backup;
628-
}
629643
}
630644
}
631645
/* skip yourself */
@@ -641,6 +655,149 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current
641655
return NULL;
642656
}
643657

658+
/*
659+
* For multi-timeline chain, look up suitable parent for incremental backup.
660+
* Multi-timeline chain has full backup and one or more descendants located
661+
* on different timelines.
662+
*/
663+
pgBackup *
664+
get_multi_timeline_parent(parray *backup_list, parray *tli_list,
665+
TimeLineID current_tli, time_t current_start_time,
666+
InstanceConfig *instance)
667+
{
668+
int i;
669+
timelineInfo *my_tlinfo = NULL;
670+
timelineInfo *tmp_tlinfo = NULL;
671+
pgBackup *ancestor_backup = NULL;
672+
673+
/* there are no timelines in the archive */
674+
if (parray_num(tli_list) == 0)
675+
return NULL;
676+
677+
/* look for current timelineInfo */
678+
for (i = 0; i < parray_num(tli_list); i++)
679+
{
680+
timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i);
681+
682+
if (tlinfo->tli == current_tli)
683+
{
684+
my_tlinfo = tlinfo;
685+
break;
686+
}
687+
}
688+
689+
if (my_tlinfo == NULL)
690+
return NULL;
691+
692+
/* Locate tlinfo of suitable full backup.
693+
* Consider this example:
694+
* t3 s2-------X <-! We are here
695+
* /
696+
* t2 s1----D---*----E--->
697+
* /
698+
* t1--A--B--*---C------->
699+
*
700+
* A, E - full backups
701+
* B, C, D - incremental backups
702+
*
703+
* We must find A.
704+
*/
705+
tmp_tlinfo = my_tlinfo;
706+
while (tmp_tlinfo->parent_link)
707+
{
708+
/* if timeline has backups, iterate over them */
709+
if (tmp_tlinfo->parent_link->backups)
710+
{
711+
for (i = 0; i < parray_num(tmp_tlinfo->parent_link->backups); i++)
712+
{
713+
pgBackup *backup = (pgBackup *) parray_get(tmp_tlinfo->parent_link->backups, i);
714+
715+
if (backup->backup_mode == BACKUP_MODE_FULL &&
716+
(backup->status == BACKUP_STATUS_OK ||
717+
backup->status == BACKUP_STATUS_DONE) &&
718+
backup->stop_lsn <= tmp_tlinfo->switchpoint)
719+
{
720+
ancestor_backup = backup;
721+
break;
722+
}
723+
}
724+
}
725+
726+
if (ancestor_backup)
727+
break;
728+
729+
tmp_tlinfo = tmp_tlinfo->parent_link;
730+
}
731+
732+
/* failed to find valid FULL backup on parent timelines */
733+
if (!ancestor_backup)
734+
return NULL;
735+
else
736+
elog(LOG, "Latest valid full backup: %s, tli: %i",
737+
base36enc(ancestor_backup->start_time), ancestor_backup->tli);
738+
739+
/* At this point we found suitable full backup,
740+
* now we must find his latest child, suitable to be
741+
* parent of current incremental backup.
742+
* Consider this example:
743+
* t3 s2-------X <-! We are here
744+
* /
745+
* t2 s1----D---*----E--->
746+
* /
747+
* t1--A--B--*---C------->
748+
*
749+
* A, E - full backups
750+
* B, C, D - incremental backups
751+
*
752+
* We found A, now we must find D.
753+
*/
754+
755+
/* Optimistically, look on current timeline for valid incremental backup, child of ancestor */
756+
if (my_tlinfo->backups)
757+
{
758+
for (i = 0; i < parray_num(my_tlinfo->backups); i++)
759+
{
760+
pgBackup *tmp_backup = NULL;
761+
pgBackup *backup = (pgBackup *) parray_get(my_tlinfo->backups, i);
762+
763+
/* found suitable parent */
764+
if (scan_parent_chain(backup, &tmp_backup) == 2 &&
765+
is_parent(ancestor_backup->start_time, backup, false))
766+
return backup;
767+
}
768+
}
769+
770+
/* Iterate over parent timelines and look for a valid backup, child of ancestor */
771+
tmp_tlinfo = my_tlinfo;
772+
while (tmp_tlinfo->parent_link)
773+
{
774+
775+
/* if timeline has backups, iterate over them */
776+
if (tmp_tlinfo->parent_link->backups)
777+
{
778+
for (i = 0; i < parray_num(tmp_tlinfo->parent_link->backups); i++)
779+
{
780+
pgBackup *tmp_backup = NULL;
781+
pgBackup *backup = (pgBackup *) parray_get(tmp_tlinfo->parent_link->backups, i);
782+
783+
/* We are not interested in backups
784+
* located outside of our timeline history
785+
*/
786+
if (backup->stop_lsn > tmp_tlinfo->switchpoint)
787+
continue;
788+
789+
if (scan_parent_chain(backup, &tmp_backup) == 2 &&
790+
is_parent(ancestor_backup->start_time, backup, true))
791+
return backup;
792+
}
793+
}
794+
795+
tmp_tlinfo = tmp_tlinfo->parent_link;
796+
}
797+
798+
return NULL;
799+
}
800+
644801
/* create backup directory in $BACKUP_PATH */
645802
int
646803
pgBackupCreateDir(pgBackup *backup)

0 commit comments

Comments
 (0)