Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 3f8c98d

Browse files
michaelpqmichaelpq
committed
pg_archivecleanup: Add --clean-backup-history
By default, pg_archivecleanup does not remove backup history files. These are just few bytes useful for debugging purposes, still keeping them around can bloat an archive path history files mixed with the WAL segments if the path has a long history. This patch adds a new option to control if backup history files are removed, depending on the oldest segment name to keep around. While on it, the TAP tests are refactored so as these are now able to handle lists of files. Each file has a flag to track if it should still exist or not depending on the oldest segment defined with the command run. Author: Atsushi Torikoshi Reviewed-by: Kyotaro Horiguchi, Fujii Masao, Michael Paquier Discussion: https://postgr.es/m/[email protected]
1 parent 4a7556f commit 3f8c98d

File tree

3 files changed

+115
-38
lines changed

3 files changed

+115
-38
lines changed

‎doc/src/sgml/ref/pgarchivecleanup.sgml

+12
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,18 @@ pg_archivecleanup: removing file "archive/00000001000000370000000E"
9393

9494
<variablelist>
9595

96+
<varlistentry>
97+
<term><option>-b</option></term>
98+
<term><option>--clean-backup-history</option></term>
99+
<listitem>
100+
<para>
101+
Remove backup history files as well.
102+
See <xref linkend="backup-base-backup"/> for details about backup
103+
history files.
104+
</para>
105+
</listitem>
106+
</varlistentry>
107+
96108
<varlistentry>
97109
<term><option>-d</option></term>
98110
<term><option>--debug</option></term>

‎src/bin/pg_archivecleanup/pg_archivecleanup.c

+16-7
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ const char *progname;
2323

2424
/* Options and defaults */
2525
bool dryrun = false; /* are we performing a dry-run operation? */
26+
bool cleanBackupHistory = false; /* remove files including backup
27+
* history files */
2628
char *additional_ext = NULL; /* Extension to remove from filenames */
2729

2830
char *archiveLocation; /* where to find the archive? */
@@ -104,18 +106,20 @@ CleanupPriorWALFiles(void)
104106
* archive */
105107

106108
/*
107-
* Truncation is essentially harmless, because we skip names of length
108-
* other than XLOG_FNAME_LEN. (In principle, one could use a
109-
* 1000-character additional_ext and get trouble.)
109+
* Truncation is essentially harmless, because we skip files whose
110+
* format is different from WAL files and backup history files. (In
111+
* principle, one could use a 1000-character additional_ext and get
112+
* trouble.)
110113
*/
111114
strlcpy(walfile, xlde->d_name, MAXPGPATH);
112115
TrimExtension(walfile, additional_ext);
113116

114117
/*
115-
* Ignore anything does that not look like a WAL segment or a .partial
116-
* WAL segment.
118+
* Ignore anything does that not look like a WAL segment, a .partial
119+
* WAL segment or a backup history file (if requested).
117120
*/
118-
if (!IsXLogFileName(walfile) && !IsPartialXLogFileName(walfile))
121+
if (!IsXLogFileName(walfile) && !IsPartialXLogFileName(walfile) &&
122+
!(cleanBackupHistory && IsBackupHistoryFileName(walfile)))
119123
continue;
120124

121125
/*
@@ -256,6 +260,7 @@ usage(void)
256260
printf(_("Usage:\n"));
257261
printf(_(" %s [OPTION]... ARCHIVELOCATION OLDESTKEPTWALFILE\n"), progname);
258262
printf(_("\nOptions:\n"));
263+
printf(_(" -b, --clean-backup-history clean up files including backup history files\n"));
259264
printf(_(" -d, --debug generate debug output (verbose mode)\n"));
260265
printf(_(" -n, --dry-run dry run, show the names of the files that would be\n"
261266
" removed\n"));
@@ -281,6 +286,7 @@ int
281286
main(int argc, char **argv)
282287
{
283288
static struct option long_options[] = {
289+
{"clean-backup-history", no_argument, NULL, 'b'},
284290
{"debug", no_argument, NULL, 'd'},
285291
{"dry-run", no_argument, NULL, 'n'},
286292
{"strip-extension", required_argument, NULL, 'x'},
@@ -306,10 +312,13 @@ main(int argc, char **argv)
306312
}
307313
}
308314

309-
while ((c = getopt_long(argc, argv, "dnx:", long_options, NULL)) != -1)
315+
while ((c = getopt_long(argc, argv, "bdnx:", long_options, NULL)) != -1)
310316
{
311317
switch (c)
312318
{
319+
case 'b': /* Remove backup history files as well */
320+
cleanBackupHistory = true;
321+
break;
313322
case 'd': /* Debug mode */
314323
pg_logging_increase_verbosity();
315324
break;

‎src/bin/pg_archivecleanup/t/010_pg_archivecleanup.pl

+87-31
Original file line numberDiff line numberDiff line change
@@ -12,22 +12,46 @@
1212

1313
my $tempdir = PostgreSQL::Test::Utils::tempdir;
1414

15-
my @walfiles = (
16-
'00000001000000370000000C.gz', '00000001000000370000000D',
17-
'00000001000000370000000E', '00000001000000370000000F.partial',);
15+
# WAL file patterns created before running each sub-scenario. "present"
16+
# tracks if the file with "name" still exists or not after running
17+
# pg_archivecleanup.
18+
my @walfiles_verbose = (
19+
{ name => '00000001000000370000000D', present => 0 },
20+
{ name => '00000001000000370000000E', present => 1 });
21+
my @walfiles_with_gz = (
22+
{ name => '00000001000000370000000C.gz', present => 0 },
23+
{ name => '00000001000000370000000D', present => 0 },
24+
{ name => '00000001000000370000000D.backup', present => 1 },
25+
{ name => '00000001000000370000000E', present => 1 },
26+
{ name => '00000001000000370000000F.partial', present => 1 },
27+
{ name => 'unrelated_file', present => 1 });
28+
my @walfiles_for_clean_backup_history = (
29+
{ name => '00000001000000370000000D', present => 0 },
30+
{ name => '00000001000000370000000D.00000028.backup', present => 0 },
31+
{ name => '00000001000000370000000E', present => 1 },
32+
{ name => '00000001000000370000000F.partial', present => 1 },
33+
{ name => 'unrelated_file', present => 1 });
1834

1935
sub create_files
2036
{
21-
foreach my $fn (@walfiles, 'unrelated_file')
37+
foreach my $fn (map { $_->{name} } @_)
2238
{
2339
open my $file, '>', "$tempdir/$fn";
40+
2441
print $file 'CONTENT';
2542
close $file;
2643
}
2744
return;
2845
}
2946

30-
create_files();
47+
sub remove_files
48+
{
49+
foreach my $fn (map { $_->{name} } @_)
50+
{
51+
unlink "$tempdir/$fn";
52+
}
53+
return;
54+
}
3155

3256
command_fails_like(
3357
['pg_archivecleanup'],
@@ -54,54 +78,86 @@ sub create_files
5478
qr/invalid file name argument/,
5579
'fails with invalid restart file name');
5680

81+
# Test a dry run, no files are physically removed, but logs are generated
82+
# to show what would be removed.
5783
{
58-
# like command_like but checking stderr
84+
create_files(@walfiles_verbose);
85+
5986
my $stderr;
87+
my $oldestkeptwalfile = '00000001000000370000000E';
6088
my $result =
6189
IPC::Run::run [ 'pg_archivecleanup', '-d', '-n', $tempdir,
62-
$walfiles[2] ],
90+
$oldestkeptwalfile ],
6391
'2>', \$stderr;
6492
ok($result, "pg_archivecleanup dry run: exit code 0");
65-
like(
66-
$stderr,
67-
qr/$walfiles[1].*would be removed/,
68-
"pg_archivecleanup dry run: matches");
69-
foreach my $fn (@walfiles)
93+
94+
for my $walpair (@walfiles_verbose)
95+
{
96+
if ($walpair->{present})
97+
{
98+
unlike(
99+
$stderr,
100+
qr/$walpair->{name}.*would be removed/,
101+
"pg_archivecleanup dry run for $walpair->{name}: matches");
102+
}
103+
else
104+
{
105+
like(
106+
$stderr,
107+
qr/$walpair->{name}.*would be removed/,
108+
"pg_archivecleanup dry run for $walpair->{name}: matches");
109+
}
110+
}
111+
foreach my $fn (map { $_->{name} } @walfiles_verbose)
70112
{
71113
ok(-f "$tempdir/$fn", "$fn not removed");
72114
}
115+
116+
remove_files(@walfiles_verbose);
73117
}
74118

75119
sub run_check
76120
{
77121
local $Test::Builder::Level = $Test::Builder::Level + 1;
78122

79-
my ($suffix, $test_name) = @_;
123+
my ($testdata, $oldestkeptwalfile, $test_name, @options) = @_;
80124

81-
create_files();
125+
create_files(@$testdata);
82126

83127
command_ok(
84-
[
85-
'pg_archivecleanup', '-x', '.gz', $tempdir,
86-
$walfiles[2] . $suffix
87-
],
128+
[ 'pg_archivecleanup', @options, $tempdir, $oldestkeptwalfile ],
88129
"$test_name: runs");
89130

90-
ok(!-f "$tempdir/$walfiles[0]",
91-
"$test_name: first older WAL file was cleaned up");
92-
ok(!-f "$tempdir/$walfiles[1]",
93-
"$test_name: second older WAL file was cleaned up");
94-
ok(-f "$tempdir/$walfiles[2]",
95-
"$test_name: restartfile was not cleaned up");
96-
ok(-f "$tempdir/$walfiles[3]",
97-
"$test_name: newer WAL file was not cleaned up");
98-
ok(-f "$tempdir/unrelated_file",
99-
"$test_name: unrelated file was not cleaned up");
131+
for my $walpair (@$testdata)
132+
{
133+
if ($walpair->{present})
134+
{
135+
ok(-f "$tempdir/$walpair->{name}",
136+
"$test_name:$walpair->{name} was not cleaned up");
137+
}
138+
else
139+
{
140+
ok(!-f "$tempdir/$walpair->{name}",
141+
"$test_name:$walpair->{name} was cleaned up");
142+
}
143+
}
144+
145+
remove_files(@$testdata);
100146
return;
101147
}
102148

103-
run_check('', 'pg_archivecleanup');
104-
run_check('.partial', 'pg_archivecleanup with .partial file');
105-
run_check('.00000020.backup', 'pg_archivecleanup with .backup file');
149+
run_check(\@walfiles_with_gz, '00000001000000370000000E',
150+
'pg_archivecleanup', '-x.gz');
151+
run_check(
152+
\@walfiles_with_gz,
153+
'00000001000000370000000E.partial',
154+
'pg_archivecleanup with .partial file', '-x.gz');
155+
run_check(
156+
\@walfiles_with_gz,
157+
'00000001000000370000000E.00000020.backup',
158+
'pg_archivecleanup with .backup file', '-x.gz');
159+
run_check(\@walfiles_for_clean_backup_history,
160+
'00000001000000370000000E',
161+
'pg_archivecleanup with --clean-backup-history', '-b');
106162

107163
done_testing();

0 commit comments

Comments
 (0)