Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 1f996ad

Browse files
tglsfdctglsfdc
committed
Fix pg_restore's direct-to-database mode for INSERT-style table data.
In commit 6545a90, I removed the mini SQL lexer that was in pg_backup_db.c, thinking that it had no real purpose beyond separating COPY data from SQL commands, which purpose had been obsoleted by long-ago fixes in pg_dump's archive file format. Unfortunately this was in error: that code was also used to identify command boundaries in INSERT-style table data, which is run together as a single string in the archive file for better compressibility. As a result, direct-to-database restores from archive files made with --inserts or --column-inserts fail in our latest releases, as reported by Dick Visser. To fix, restore the mini SQL lexer, but simplify it by adjusting the calling logic so that it's only required to cope with INSERT-style table data, not arbitrary SQL commands. This allows us to not have to deal with SQL comments, E'' strings, or dollar-quoted strings, none of which have ever been emitted by dumpTableData_insert. Also, fix the lexer to cope with standard-conforming strings, which was the actual bug that the previous patch was meant to solve. Back-patch to all supported branches. The previous patch went back to 8.2, which unfortunately means that the EOL release of 8.2 contains this bug, but I don't think we're doing another 8.2 release just because of that.
1 parent c024a3b commit 1f996ad

File tree

4 files changed

+135
-11
lines changed

4 files changed

+135
-11
lines changed

‎src/bin/pg_dump/pg_backup_archiver.c

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -607,20 +607,20 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te,
607607
if (te->copyStmt && strlen(te->copyStmt) > 0)
608608
{
609609
ahprintf(AH, "%s", te->copyStmt);
610-
AH->writingCopyData = true;
610+
AH->outputKind = OUTPUT_COPYDATA;
611611
}
612+
else
613+
AH->outputKind = OUTPUT_OTHERDATA;
612614

613615
(*AH->PrintTocDataPtr) (AH, te, ropt);
614616

615617
/*
616618
* Terminate COPY if needed.
617619
*/
618-
if (AH->writingCopyData)
619-
{
620-
if (RestoringToDB(AH))
621-
EndDBCopyMode(AH, te);
622-
AH->writingCopyData = false;
623-
}
620+
if (AH->outputKind == OUTPUT_COPYDATA &&
621+
RestoringToDB(AH))
622+
EndDBCopyMode(AH, te);
623+
AH->outputKind = OUTPUT_SQLCMDS;
624624

625625
/* close out the transaction started above */
626626
if (is_parallel && te->created)
@@ -1955,6 +1955,8 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
19551955
AH->mode = mode;
19561956
AH->compression = compression;
19571957

1958+
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
1959+
19581960
/* Open stdout with no compression for AH output handle */
19591961
AH->gzOut = 0;
19601962
AH->OF = stdout;
@@ -4144,7 +4146,8 @@ CloneArchive(ArchiveHandle *AH)
41444146
die_horribly(AH, modulename, "out of memory\n");
41454147
memcpy(clone, AH, sizeof(ArchiveHandle));
41464148

4147-
/* Handle format-independent fields ... none at the moment */
4149+
/* Handle format-independent fields */
4150+
memset(&(clone->sqlparse), 0, sizeof(clone->sqlparse));
41484151

41494152
/* The clone will have its own connection, so disregard connection state */
41504153
clone->connection = NULL;
@@ -4177,7 +4180,9 @@ DeCloneArchive(ArchiveHandle *AH)
41774180
/* Clear format-specific state */
41784181
(AH->DeClonePtr) (AH);
41794182

4180-
/* Clear state allocated by CloneArchive ... none at the moment */
4183+
/* Clear state allocated by CloneArchive */
4184+
if (AH->sqlparse.curCmd)
4185+
destroyPQExpBuffer(AH->sqlparse.curCmd);
41814186

41824187
/* Clear any connection-local state */
41834188
if (AH->currUser)

‎src/bin/pg_dump/pg_backup_archiver.h

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,20 @@ typedef struct _outputContext
137137
int gzOut;
138138
} OutputContext;
139139

140+
typedef enum
141+
{
142+
SQL_SCAN = 0, /* normal */
143+
SQL_IN_SINGLE_QUOTE, /* '...' literal */
144+
SQL_IN_DOUBLE_QUOTE /* "..." identifier */
145+
} sqlparseState;
146+
147+
typedef struct
148+
{
149+
sqlparseState state; /* see above */
150+
bool backSlash; /* next char is backslash quoted? */
151+
PQExpBuffer curCmd; /* incomplete line (NULL if not created) */
152+
} sqlparseInfo;
153+
140154
typedef enum
141155
{
142156
STAGE_NONE = 0,
@@ -145,6 +159,13 @@ typedef enum
145159
STAGE_FINALIZING
146160
} ArchiverStage;
147161

162+
typedef enum
163+
{
164+
OUTPUT_SQLCMDS = 0, /* emitting general SQL commands */
165+
OUTPUT_COPYDATA, /* writing COPY data */
166+
OUTPUT_OTHERDATA /* writing data as INSERT commands */
167+
} ArchiverOutput;
168+
148169
typedef enum
149170
{
150171
REQ_SCHEMA = 1,
@@ -172,6 +193,8 @@ typedef struct _archiveHandle
172193
* Added V1.7 */
173194
ArchiveFormat format; /* Archive format */
174195

196+
sqlparseInfo sqlparse; /* state for parsing INSERT data */
197+
175198
time_t createDate; /* Date archive created */
176199

177200
/*
@@ -222,7 +245,7 @@ typedef struct _archiveHandle
222245
PGconn *connection;
223246
int connectToDB; /* Flag to indicate if direct DB connection is
224247
* required */
225-
bool writingCopyData; /* True when we are sending COPY data */
248+
ArchiverOutput outputKind; /* Flag for what we're currently writing */
226249
bool pgCopyIn; /* Currently in libpq 'COPY IN' mode. */
227250

228251
int loFd; /* BLOB fd */

‎src/bin/pg_dump/pg_backup_db.c

Lines changed: 89 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -369,15 +369,93 @@ ExecuteSqlCommand(ArchiveHandle *AH, const char *qry, const char *desc)
369369
}
370370

371371

372+
/*
373+
* Process non-COPY table data (that is, INSERT commands).
374+
*
375+
* The commands have been run together as one long string for compressibility,
376+
* and we are receiving them in bufferloads with arbitrary boundaries, so we
377+
* have to locate command boundaries and save partial commands across calls.
378+
* All state must be kept in AH->sqlparse, not in local variables of this
379+
* routine. We assume that AH->sqlparse was filled with zeroes when created.
380+
*
381+
* We have to lex the data to the extent of identifying literals and quoted
382+
* identifiers, so that we can recognize statement-terminating semicolons.
383+
* We assume that INSERT data will not contain SQL comments, E'' literals,
384+
* or dollar-quoted strings, so this is much simpler than a full SQL lexer.
385+
*/
386+
static void
387+
ExecuteInsertCommands(ArchiveHandle *AH, const char *buf, size_t bufLen)
388+
{
389+
const char *qry = buf;
390+
const char *eos = buf + bufLen;
391+
392+
/* initialize command buffer if first time through */
393+
if (AH->sqlparse.curCmd == NULL)
394+
AH->sqlparse.curCmd = createPQExpBuffer();
395+
396+
for (; qry < eos; qry++)
397+
{
398+
char ch = *qry;
399+
400+
/* For neatness, we skip any newlines between commands */
401+
if (!(ch == '\n' && AH->sqlparse.curCmd->len == 0))
402+
appendPQExpBufferChar(AH->sqlparse.curCmd, ch);
403+
404+
switch (AH->sqlparse.state)
405+
{
406+
case SQL_SCAN: /* Default state == 0, set in _allocAH */
407+
if (ch == ';')
408+
{
409+
/*
410+
* We've found the end of a statement. Send it and reset
411+
* the buffer.
412+
*/
413+
ExecuteSqlCommand(AH, AH->sqlparse.curCmd->data,
414+
"could not execute query");
415+
resetPQExpBuffer(AH->sqlparse.curCmd);
416+
}
417+
else if (ch == '\'')
418+
{
419+
AH->sqlparse.state = SQL_IN_SINGLE_QUOTE;
420+
AH->sqlparse.backSlash = false;
421+
}
422+
else if (ch == '"')
423+
{
424+
AH->sqlparse.state = SQL_IN_DOUBLE_QUOTE;
425+
}
426+
break;
427+
428+
case SQL_IN_SINGLE_QUOTE:
429+
/* We needn't handle '' specially */
430+
if (ch == '\'' && !AH->sqlparse.backSlash)
431+
AH->sqlparse.state = SQL_SCAN;
432+
else if (ch == '\\' && !AH->public.std_strings)
433+
AH->sqlparse.backSlash = !AH->sqlparse.backSlash;
434+
else
435+
AH->sqlparse.backSlash = false;
436+
break;
437+
438+
case SQL_IN_DOUBLE_QUOTE:
439+
/* We needn't handle "" specially */
440+
if (ch == '"')
441+
AH->sqlparse.state = SQL_SCAN;
442+
break;
443+
}
444+
}
445+
}
446+
447+
372448
/*
373449
* Implement ahwrite() for direct-to-DB restore
374450
*/
375451
int
376452
ExecuteSqlCommandBuf(ArchiveHandle *AH, const char *buf, size_t bufLen)
377453
{
378-
if (AH->writingCopyData)
454+
if (AH->outputKind == OUTPUT_COPYDATA)
379455
{
380456
/*
457+
* COPY data.
458+
*
381459
* We drop the data on the floor if libpq has failed to enter COPY
382460
* mode; this allows us to behave reasonably when trying to continue
383461
* after an error in a COPY command.
@@ -387,9 +465,19 @@ ExecuteSqlCommandBuf(ArchiveHandle *AH, const char *buf, size_t bufLen)
387465
die_horribly(AH, modulename, "error returned by PQputCopyData: %s",
388466
PQerrorMessage(AH->connection));
389467
}
468+
else if (AH->outputKind == OUTPUT_OTHERDATA)
469+
{
470+
/*
471+
* Table data expressed as INSERT commands.
472+
*/
473+
ExecuteInsertCommands(AH, buf, bufLen);
474+
}
390475
else
391476
{
392477
/*
478+
* General SQL commands; we assume that commands will not be split
479+
* across calls.
480+
*
393481
* In most cases the data passed to us will be a null-terminated
394482
* string, but if it's not, we have to add a trailing null.
395483
*/

‎src/bin/pg_dump/pg_dump.c

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1243,6 +1243,14 @@ dumpTableData_copy(Archive *fout, void *dcontext)
12431243
return 1;
12441244
}
12451245

1246+
/*
1247+
* Dump table data using INSERT commands.
1248+
*
1249+
* Caution: when we restore from an archive file direct to database, the
1250+
* INSERT commands emitted by this function have to be parsed by
1251+
* pg_backup_db.c's ExecuteInsertCommands(), which will not handle comments,
1252+
* E'' strings, or dollar-quoted strings. So don't emit anything like that.
1253+
*/
12461254
static int
12471255
dumpTableData_insert(Archive *fout, void *dcontext)
12481256
{

0 commit comments

Comments
 (0)