Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 0c7c78b

Browse files
funny-falconfunny-falcon
committed
Merge branch 'REL_2_5-PBCKP-146-2' into REL_2_5
2 parents 34af025 + e36924a commit 0c7c78b

File tree

12 files changed

+727
-256
lines changed

12 files changed

+727
-256
lines changed

‎src/archive.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1375,11 +1375,11 @@ get_wal_file(const char *filename, const char *from_fullpath,
13751375
#ifdef HAVE_LIBZ
13761376
/* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */
13771377
if (IsXLogFileName(filename))
1378-
rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, &errmsg);
1378+
rc = fio_send_file_gz(from_fullpath_gz, out, &errmsg);
13791379
if (rc == FILE_MISSING)
13801380
#endif
13811381
/* ... failing that, use uncompressed */
1382-
rc = fio_send_file(from_fullpath, to_fullpath, out, NULL, &errmsg);
1382+
rc = fio_send_file(from_fullpath, out, false, NULL, &errmsg);
13831383

13841384
/* When not in prefetch mode, try to use partial file */
13851385
if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename))
@@ -1389,13 +1389,13 @@ get_wal_file(const char *filename, const char *from_fullpath,
13891389
#ifdef HAVE_LIBZ
13901390
/* '.gz.partial' goes first ... */
13911391
snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath);
1392-
rc = fio_send_file_gz(from_partial, to_fullpath, out, &errmsg);
1392+
rc = fio_send_file_gz(from_partial, out, &errmsg);
13931393
if (rc == FILE_MISSING)
13941394
#endif
13951395
{
13961396
/* ... failing that, use '.partial' */
13971397
snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath);
1398-
rc = fio_send_file(from_partial, to_fullpath, out, NULL, &errmsg);
1398+
rc = fio_send_file(from_partial, out, false, NULL, &errmsg);
13991399
}
14001400

14011401
if (rc == SEND_OK)

‎src/catalog.c

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1069,6 +1069,7 @@ get_backup_filelist(pgBackup *backup, bool strict)
10691069
char linked[MAXPGPATH];
10701070
char compress_alg_string[MAXPGPATH];
10711071
int64 write_size,
1072+
uncompressed_size,
10721073
mode, /* bit length of mode_t depends on platforms */
10731074
is_datafile,
10741075
is_cfs,
@@ -1132,6 +1133,11 @@ get_backup_filelist(pgBackup *backup, bool strict)
11321133
if (get_control_value_int64(buf, "hdr_size", &hdr_size, false))
11331134
file->hdr_size = (int) hdr_size;
11341135

1136+
if (get_control_value_int64(buf, "full_size", &uncompressed_size, false))
1137+
file->uncompressed_size = uncompressed_size;
1138+
else
1139+
file->uncompressed_size = write_size;
1140+
11351141
if (file->external_dir_num == 0)
11361142
set_forkname(file);
11371143

@@ -2561,6 +2567,11 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
25612567
file->external_dir_num,
25622568
file->dbOid);
25632569

2570+
if (file->uncompressed_size != 0 &&
2571+
file->uncompressed_size != file->write_size)
2572+
len += sprintf(line+len, ",\"full_size\":\"" INT64_FORMAT "\"",
2573+
file->uncompressed_size);
2574+
25642575
if (file->is_datafile)
25652576
len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno);
25662577

‎src/data.c

Lines changed: 36 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -799,6 +799,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
799799
* and its mtime is less than parent backup start time ... */
800800
if ((pg_strcasecmp(file->name, RELMAPPER_FILENAME) != 0) &&
801801
(prev_file && file->exists_in_prev &&
802+
file->size == prev_file->size &&
802803
file->mtime <= parent_backup_time))
803804
{
804805
/*
@@ -1330,7 +1331,12 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
13301331
if (already_exists)
13311332
{
13321333
/* compare checksums of already existing file and backup file */
1333-
pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false);
1334+
pg_crc32 file_crc;
1335+
if (tmp_file->forkName == cfm &&
1336+
tmp_file->uncompressed_size > tmp_file->write_size)
1337+
file_crc = fio_get_crc32_truncated(to_fullpath, FIO_DB_HOST);
1338+
else
1339+
file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false);
13341340

13351341
if (file_crc == tmp_file->crc)
13361342
{
@@ -1387,10 +1393,12 @@ backup_non_data_file_internal(const char *from_fullpath,
13871393
const char *to_fullpath, pgFile *file,
13881394
bool missing_ok)
13891395
{
1390-
FILE *in = NULL;
13911396
FILE *out = NULL;
1392-
ssize_t read_len = 0;
1393-
char *buf = NULL;
1397+
char *errmsg = NULL;
1398+
int rc;
1399+
bool cut_zero_tail;
1400+
1401+
cut_zero_tail = file->forkName == cfm;
13941402

13951403
INIT_FILE_CRC32(true, file->crc);
13961404

@@ -1412,107 +1420,44 @@ backup_non_data_file_internal(const char *from_fullpath,
14121420

14131421
/* backup remote file */
14141422
if (fio_is_remote(FIO_DB_HOST))
1415-
{
1416-
char *errmsg = NULL;
1417-
int rc = fio_send_file(from_fullpath, to_fullpath, out, file, &errmsg);
1423+
rc = fio_send_file(from_fullpath, out, cut_zero_tail, file, &errmsg);
1424+
else
1425+
rc = fio_send_file_local(from_fullpath, out, cut_zero_tail, file, &errmsg);
14181426

1419-
/* handle errors */
1420-
if (rc == FILE_MISSING)
1421-
{
1422-
/* maybe deleted, it's not error in case of backup */
1423-
if (missing_ok)
1424-
{
1425-
elog(LOG, "File \"%s\" is not found", from_fullpath);
1426-
file->write_size = FILE_NOT_FOUND;
1427-
goto cleanup;
1428-
}
1429-
else
1430-
elog(ERROR, "File \"%s\" is not found", from_fullpath);
1431-
}
1432-
else if (rc == WRITE_FAILED)
1433-
elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno));
1434-
else if (rc != SEND_OK)
1427+
/* handle errors */
1428+
if (rc == FILE_MISSING)
1429+
{
1430+
/* maybe deleted, it's not error in case of backup */
1431+
if (missing_ok)
14351432
{
1436-
if (errmsg)
1437-
elog(ERROR, "%s", errmsg);
1438-
else
1439-
elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath);
1433+
elog(LOG, "File \"%s\" is not found", from_fullpath);
1434+
file->write_size = FILE_NOT_FOUND;
1435+
goto cleanup;
14401436
}
1441-
1442-
pg_free(errmsg);
1437+
else
1438+
elog(ERROR, "File \"%s\" is not found", from_fullpath);
14431439
}
1444-
/* backup local file */
1445-
else
1440+
else if (rc == WRITE_FAILED)
1441+
elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno));
1442+
else if (rc != SEND_OK)
14461443
{
1447-
/* open source file for read */
1448-
in = fopen(from_fullpath, PG_BINARY_R);
1449-
if (in == NULL)
1450-
{
1451-
/* maybe deleted, it's not error in case of backup */
1452-
if (errno == ENOENT)
1453-
{
1454-
if (missing_ok)
1455-
{
1456-
elog(LOG, "File \"%s\" is not found", from_fullpath);
1457-
file->write_size = FILE_NOT_FOUND;
1458-
goto cleanup;
1459-
}
1460-
else
1461-
elog(ERROR, "File \"%s\" is not found", from_fullpath);
1462-
}
1463-
1464-
elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath,
1465-
strerror(errno));
1466-
}
1467-
1468-
/* disable stdio buffering for local input/output files to avoid triple buffering */
1469-
setvbuf(in, NULL, _IONBF, BUFSIZ);
1470-
setvbuf(out, NULL, _IONBF, BUFSIZ);
1471-
1472-
/* allocate 64kB buffer */
1473-
buf = pgut_malloc(CHUNK_SIZE);
1474-
1475-
/* copy content and calc CRC */
1476-
for (;;)
1477-
{
1478-
read_len = fread(buf, 1, CHUNK_SIZE, in);
1479-
1480-
if (ferror(in))
1481-
elog(ERROR, "Cannot read from file \"%s\": %s",
1482-
from_fullpath, strerror(errno));
1483-
1484-
if (read_len > 0)
1485-
{
1486-
if (fwrite(buf, 1, read_len, out) != read_len)
1487-
elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath,
1488-
strerror(errno));
1489-
1490-
/* update CRC */
1491-
COMP_FILE_CRC32(true, file->crc, buf, read_len);
1492-
file->read_size += read_len;
1493-
}
1494-
1495-
if (feof(in))
1496-
break;
1497-
}
1444+
if (errmsg)
1445+
elog(ERROR, "%s", errmsg);
1446+
else
1447+
elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath);
14981448
}
14991449

1500-
file->write_size = (int64) file->read_size;
1501-
1502-
if (file->write_size > 0)
1503-
file->uncompressed_size = file->write_size;
1450+
file->uncompressed_size = file->read_size;
15041451

15051452
cleanup:
1453+
if (errmsg != NULL)
1454+
pg_free(errmsg);
1455+
15061456
/* finish CRC calculation and store into pgFile */
15071457
FIN_FILE_CRC32(true, file->crc);
15081458

1509-
if (in && fclose(in))
1510-
elog(ERROR, "Cannot close the file \"%s\": %s", from_fullpath, strerror(errno));
1511-
15121459
if (out && fclose(out))
15131460
elog(ERROR, "Cannot close the file \"%s\": %s", to_fullpath, strerror(errno));
1514-
1515-
pg_free(buf);
15161461
}
15171462

15181463
/*

‎src/dir.c

Lines changed: 1 addition & 132 deletions
Original file line numberDiff line numberDiff line change
@@ -262,137 +262,6 @@ pgFileDelete(mode_t mode, const char *full_path)
262262
}
263263
}
264264

265-
/*
266-
* Read the local file to compute its CRC.
267-
* We cannot make decision about file decompression because
268-
* user may ask to backup already compressed files and we should be
269-
* obvious about it.
270-
*/
271-
pg_crc32
272-
pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok)
273-
{
274-
FILE *fp;
275-
pg_crc32 crc = 0;
276-
char *buf;
277-
size_t len = 0;
278-
279-
INIT_FILE_CRC32(use_crc32c, crc);
280-
281-
/* open file in binary read mode */
282-
fp = fopen(file_path, PG_BINARY_R);
283-
if (fp == NULL)
284-
{
285-
if (errno == ENOENT)
286-
{
287-
if (missing_ok)
288-
{
289-
FIN_FILE_CRC32(use_crc32c, crc);
290-
return crc;
291-
}
292-
}
293-
294-
elog(ERROR, "Cannot open file \"%s\": %s",
295-
file_path, strerror(errno));
296-
}
297-
298-
/* disable stdio buffering */
299-
setvbuf(fp, NULL, _IONBF, BUFSIZ);
300-
buf = pgut_malloc(STDIO_BUFSIZE);
301-
302-
/* calc CRC of file */
303-
for (;;)
304-
{
305-
if (interrupted)
306-
elog(ERROR, "interrupted during CRC calculation");
307-
308-
len = fread(buf, 1, STDIO_BUFSIZE, fp);
309-
310-
if (ferror(fp))
311-
elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno));
312-
313-
/* update CRC */
314-
COMP_FILE_CRC32(use_crc32c, crc, buf, len);
315-
316-
if (feof(fp))
317-
break;
318-
}
319-
320-
FIN_FILE_CRC32(use_crc32c, crc);
321-
fclose(fp);
322-
pg_free(buf);
323-
324-
return crc;
325-
}
326-
327-
/*
328-
* Read the local file to compute its CRC.
329-
* We cannot make decision about file decompression because
330-
* user may ask to backup already compressed files and we should be
331-
* obvious about it.
332-
*/
333-
pg_crc32
334-
pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok)
335-
{
336-
gzFile fp;
337-
pg_crc32 crc = 0;
338-
int len = 0;
339-
int err;
340-
char *buf;
341-
342-
INIT_FILE_CRC32(use_crc32c, crc);
343-
344-
/* open file in binary read mode */
345-
fp = gzopen(file_path, PG_BINARY_R);
346-
if (fp == NULL)
347-
{
348-
if (errno == ENOENT)
349-
{
350-
if (missing_ok)
351-
{
352-
FIN_FILE_CRC32(use_crc32c, crc);
353-
return crc;
354-
}
355-
}
356-
357-
elog(ERROR, "Cannot open file \"%s\": %s",
358-
file_path, strerror(errno));
359-
}
360-
361-
buf = pgut_malloc(STDIO_BUFSIZE);
362-
363-
/* calc CRC of file */
364-
for (;;)
365-
{
366-
if (interrupted)
367-
elog(ERROR, "interrupted during CRC calculation");
368-
369-
len = gzread(fp, buf, STDIO_BUFSIZE);
370-
371-
if (len <= 0)
372-
{
373-
/* we either run into eof or error */
374-
if (gzeof(fp))
375-
break;
376-
else
377-
{
378-
const char *err_str = NULL;
379-
380-
err_str = gzerror(fp, &err);
381-
elog(ERROR, "Cannot read from compressed file %s", err_str);
382-
}
383-
}
384-
385-
/* update CRC */
386-
COMP_FILE_CRC32(use_crc32c, crc, buf, len);
387-
}
388-
389-
FIN_FILE_CRC32(use_crc32c, crc);
390-
gzclose(fp);
391-
pg_free(buf);
392-
393-
return crc;
394-
}
395-
396265
void
397266
pgFileFree(void *file)
398267
{
@@ -1812,7 +1681,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_
18121681
FIO_BACKUP_HOST);
18131682
file->crc = pgFileGetCRC(database_map_path, true, false);
18141683
file->write_size = file->size;
1815-
file->uncompressed_size = file->read_size;
1684+
file->uncompressed_size = file->size;
18161685

18171686
parray_append(backup_files_list, file);
18181687
}

‎src/merge.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1078,7 +1078,7 @@ merge_files(void *arg)
10781078
tmp_file->hdr_crc = file->hdr_crc;
10791079
}
10801080
else
1081-
tmp_file->uncompressed_size = tmp_file->write_size;
1081+
tmp_file->uncompressed_size = tmp_file->uncompressed_size;
10821082

10831083
/* Copy header metadata from old map into a new one */
10841084
tmp_file->n_headers = file->n_headers;

0 commit comments

Comments
 (0)