Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 6a824d9

Browse files
nathan-bossartCommitfest Bot
nathan-bossart
authored and
Commitfest Bot
committed
pg_dump: Reduce memory usage of dumps with statistics.
Right now, pg_dump stores all generated commands for statistics in memory. These commands can be quite large and therefore can significantly increase pg_dump's memory footprint. To fix, wait until we are about to write out the commands before generating them, and be sure to free the commands after writing. This is implemented via a new defnDumper callback that works much like the dataDumper one but is specially designed for TOC entries. Custom dumps that include data might write the TOC twice (to update data offset information), which would ordinarily cause pg_dump to run the attribute statistics queries twice. However, as a hack, we save the length of the written-out entry in the first pass, and we skip over it in the second. While there is no known technical problem with executing the queries multiple times and rewriting the results, it's expensive and feels risky, so it seems prudent to avoid it. As an exception, we _do_ execute the queries twice for the tar format. This format does a second pass through the TOC to generate the restore.sql file, which isn't used by pg_restore, so different results won't corrupt the output (it'll just be different). We could alternatively save the definition in memory the first time it is generated, but that defeats the purpose of this change. In any case, past discussion indicates that the tar format might be a candidate for deprecation, so it doesn't seem worth trying too much harder. Author: Corey Huinker <[email protected]> Co-authored-by: Nathan Bossart <[email protected]> Reviewed-by: Jeff Davis <[email protected]> Discussion: https://postgr.es/m/CADkLM%3Dc%2Br05srPy9w%2B-%2BnbmLEo15dKXYQ03Q_xyK%2BriJerigLQ%40mail.gmail.com
1 parent 07b47f1 commit 6a824d9

File tree

4 files changed

+114
-16
lines changed

4 files changed

+114
-16
lines changed

‎src/bin/pg_dump/pg_backup.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -285,6 +285,7 @@ typedef int DumpId;
285285
* Function pointer prototypes for assorted callback methods.
286286
*/
287287

288+
typedef char *(*DefnDumperPtr) (Archive *AH, const void *userArg);
288289
typedef int (*DataDumperPtr) (Archive *AH, const void *userArg);
289290

290291
typedef void (*SetupWorkerPtrType) (Archive *AH);

‎src/bin/pg_dump/pg_backup_archiver.c‎

Lines changed: 75 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1266,6 +1266,9 @@ ArchiveEntry(Archive *AHX, CatalogId catalogId, DumpId dumpId,
12661266
newToc->dataDumperArg = opts->dumpArg;
12671267
newToc->hadDumper = opts->dumpFn ? true : false;
12681268

1269+
newToc->defnDumper = opts->defnFn;
1270+
newToc->defnDumperArg = opts->defnArg;
1271+
12691272
newToc->formatData = NULL;
12701273
newToc->dataLength = 0;
12711274

@@ -2621,7 +2624,40 @@ WriteToc(ArchiveHandle *AH)
26212624
WriteStr(AH, te->tag);
26222625
WriteStr(AH, te->desc);
26232626
WriteInt(AH, te->section);
2624-
WriteStr(AH, te->defn);
2627+
2628+
if (te->defnLen)
2629+
{
2630+
/*
2631+
* defnLen should only be set for custom format's second call to
2632+
* WriteToc(), which rewrites the TOC in place to update any data
2633+
* offsets. Rather than call the defnDumper a second time (which
2634+
* would involve executing the queries again), just skip writing
2635+
* the entry. While regenerating the definition should in theory
2636+
* produce the same result as before, it's expensive and feels
2637+
* risky.
2638+
*
2639+
* The custom format only does a second WriteToc() if fseeko() is
2640+
* usable (see _CloseArchive() in pg_backup_custom.c), so we can
2641+
* just use it without checking. For other formats, we fail
2642+
* because this assumption must no longer hold true.
2643+
*/
2644+
if (AH->format != archCustom)
2645+
pg_fatal("unexpected TOC entry in WriteToc(): %d %s %s",
2646+
te->dumpId, te->desc, te->tag);
2647+
2648+
if (fseeko(AH->FH, te->defnLen, SEEK_CUR != 0))
2649+
pg_fatal("error during file seek: %m");
2650+
}
2651+
else if (te->defnDumper)
2652+
{
2653+
char *defn = te->defnDumper((Archive *) AH, te->defnDumperArg);
2654+
2655+
te->defnLen = WriteStr(AH, defn);
2656+
pg_free(defn);
2657+
}
2658+
else
2659+
WriteStr(AH, te->defn);
2660+
26252661
WriteStr(AH, te->dropStmt);
26262662
WriteStr(AH, te->copyStmt);
26272663
WriteStr(AH, te->namespace);
@@ -3849,7 +3885,7 @@ _printTocEntry(ArchiveHandle *AH, TocEntry *te, const char *pfx)
38493885

38503886
/*
38513887
* Actually print the definition. Normally we can just print the defn
3852-
* string if any, but we have three special cases:
3888+
* string if any, but we have four special cases:
38533889
*
38543890
* 1. A crude hack for suppressing AUTHORIZATION clause that old pg_dump
38553891
* versions put into CREATE SCHEMA. Don't mutate the variant for schema
@@ -3862,6 +3898,10 @@ _printTocEntry(ArchiveHandle *AH, TocEntry *te, const char *pfx)
38623898
* 3. ACL LARGE OBJECTS entries need special processing because they
38633899
* contain only one copy of the ACL GRANT/REVOKE commands, which we must
38643900
* apply to each large object listed in the associated BLOB METADATA.
3901+
*
3902+
* 4. Entries with a defnDumper need to call it to generate the
3903+
* definition. This is primarily intended to provide a way to save memory
3904+
* for objects that need a lot of it (e.g., statistics data).
38653905
*/
38663906
if (ropt->noOwner &&
38673907
strcmp(te->desc, "SCHEMA") == 0 && strncmp(te->defn, "--", 2) != 0)
@@ -3877,6 +3917,39 @@ _printTocEntry(ArchiveHandle *AH, TocEntry *te, const char *pfx)
38773917
{
38783918
IssueACLPerBlob(AH, te);
38793919
}
3920+
else if (te->defnLen && AH->format != archTar)
3921+
{
3922+
/*
3923+
* If defnLen is set, the defnDumper has already been called for this
3924+
* TOC entry. We ordinarily don't expect a defnDumper to be called on
3925+
* a TOC entry a second time in _printTocEntry(), but of course
3926+
* there's an exception. The tar format first calls WriteToc(), which
3927+
* scans through the entire TOC, and then it later calls
3928+
* RestoreArchive() to generate restore.sql, which scans through the
3929+
* TOC again. There doesn't seem to be a good way to avoid calling the
3930+
* defnDumper again in that case without storing the definition in
3931+
* memory, which is what we're trying to avoid in the first place.
3932+
* This second defnDumper invocation should generate the exact same
3933+
* output, but even if it doesn't, the worst case is that the
3934+
* restore.sql file (which isn't used by pg_restore) is incorrect.
3935+
* Past discussion on the mailing list indicates that tar format isn't
3936+
* known to be heavily used and might be a candidate for deprecation,
3937+
* so it doesn't seem worth trying much harder here.
3938+
*
3939+
* In all other cases, encountering a TOC entry a second time in
3940+
* _printTocEntry() is unexpected, so we fail because one of our
3941+
* assumptions must no longer hold true.
3942+
*/
3943+
pg_fatal("unexpected TOC entry in _printTocEntry(): %d %s %s",
3944+
te->dumpId, te->desc, te->tag);
3945+
}
3946+
else if (te->defnDumper)
3947+
{
3948+
char *defn = te->defnDumper((Archive *) AH, te->defnDumperArg);
3949+
3950+
te->defnLen = ahprintf(AH, "%s\n\n", defn);
3951+
pg_free(defn);
3952+
}
38803953
else if (te->defn && strlen(te->defn) > 0)
38813954
{
38823955
ahprintf(AH, "%s\n\n", te->defn);

‎src/bin/pg_dump/pg_backup_archiver.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -368,6 +368,10 @@ struct _tocEntry
368368
const void *dataDumperArg; /* Arg for above routine */
369369
void *formatData; /* TOC Entry data specific to file format */
370370

371+
DefnDumperPtr defnDumper; /* routine to dump definition statement */
372+
const void *defnDumperArg; /* arg for above routine */
373+
size_t defnLen; /* length of dumped definition */
374+
371375
/* working state while dumping/restoring */
372376
pgoff_t dataLength; /* item's data size; 0 if none or unknown */
373377
int reqs; /* do we need schema and/or data of object
@@ -407,6 +411,8 @@ typedef struct _archiveOpts
407411
int nDeps;
408412
DataDumperPtr dumpFn;
409413
const void *dumpArg;
414+
DefnDumperPtr defnFn;
415+
const void *defnArg;
410416
} ArchiveOpts;
411417
#define ARCHIVE_OPTS(...) &(ArchiveOpts){__VA_ARGS__}
412418
/* Called to add a TOC entry */

‎src/bin/pg_dump/pg_dump.c‎

Lines changed: 32 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10554,17 +10554,21 @@ appendNamedArgument(PQExpBuffer out, Archive *fout, const char *argname,
1055410554
}
1055510555

1055610556
/*
10557-
* dumpRelationStats --
10557+
* dumpRelationStats_dumper --
1055810558
*
10559-
* Dump command to import stats into the relation on the new database.
10559+
* Generate command to import stats into the relation on the new database.
10560+
* This routine is called by the Archiver when it wants the statistics to be
10561+
* dumped.
1056010562
*/
10561-
static void
10562-
dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
10563+
static char *
10564+
dumpRelationStats_dumper(Archive *fout, const void *userArg)
1056310565
{
10566+
const RelStatsInfo *rsinfo = (RelStatsInfo *) userArg;
1056410567
const DumpableObject *dobj = &rsinfo->dobj;
1056510568
PGresult *res;
1056610569
PQExpBuffer query;
10567-
PQExpBuffer out;
10570+
PQExpBufferData out_data;
10571+
PQExpBuffer out = &out_data;
1056810572
int i_attname;
1056910573
int i_inherited;
1057010574
int i_null_frac;
@@ -10581,10 +10585,6 @@ dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
1058110585
int i_range_empty_frac;
1058210586
int i_range_bounds_histogram;
1058310587

10584-
/* nothing to do if we are not dumping statistics */
10585-
if (!fout->dopt->dumpStatistics)
10586-
return;
10587-
1058810588
query = createPQExpBuffer();
1058910589
if (!fout->is_prepared[PREPQUERY_GETATTRIBUTESTATS])
1059010590
{
@@ -10620,7 +10620,7 @@ dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
1062010620
resetPQExpBuffer(query);
1062110621
}
1062210622

10623-
out = createPQExpBuffer();
10623+
initPQExpBuffer(out);
1062410624

1062510625
/* restore relation stats */
1062610626
appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_relation_stats(\n");
@@ -10764,17 +10764,35 @@ dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
1076410764

1076510765
PQclear(res);
1076610766

10767+
destroyPQExpBuffer(query);
10768+
return out->data;
10769+
}
10770+
10771+
/*
10772+
* dumpRelationStats --
10773+
*
10774+
* Make an ArchiveEntry for the relation statistics. The Archiver will take
10775+
* care of gathering the statistics and generating the restore commands when
10776+
* they are needed.
10777+
*/
10778+
static void
10779+
dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
10780+
{
10781+
const DumpableObject *dobj = &rsinfo->dobj;
10782+
10783+
/* nothing to do if we are not dumping statistics */
10784+
if (!fout->dopt->dumpStatistics)
10785+
return;
10786+
1076710787
ArchiveEntry(fout, nilCatalogId, createDumpId(),
1076810788
ARCHIVE_OPTS(.tag = dobj->name,
1076910789
.namespace = dobj->namespace->dobj.name,
1077010790
.description = "STATISTICS DATA",
1077110791
.section = rsinfo->section,
10772-
.createStmt = out->data,
10792+
.defnFn = dumpRelationStats_dumper,
10793+
.defnArg = rsinfo,
1077310794
.deps = dobj->dependencies,
1077410795
.nDeps = dobj->nDeps));
10775-
10776-
destroyPQExpBuffer(out);
10777-
destroyPQExpBuffer(query);
1077810796
}
1077910797

1078010798
/*

0 commit comments

Comments
 (0)