Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e6de9e4

Browse files
knizhnikknizhnik
committed
First working version of backup with encryption
1 parent 42ba0f8 commit e6de9e4

File tree

11 files changed

+107
-43
lines changed

11 files changed

+107
-43
lines changed

‎src/archive.c

+3-3
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress,
187187
else
188188
#endif
189189
{
190-
int out_fd = -1;
190+
int out_fd;
191191
snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path);
192192

193193
out_fd = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST);
@@ -531,11 +531,11 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed)
531531
else
532532
#endif
533533
{
534-
crc2 = pgFileGetCRC(path2, true, true, NULL, FIO_BACKUP_HOST);
534+
crc2 = pgFileGetCRC(path2, true, true, NULL, FIO_BACKUP_HOST, instance_config.encryption);
535535
}
536536

537537
/* Get checksum of original file */
538-
crc1 = pgFileGetCRC(path1, true, true, NULL, FIO_DB_HOST);
538+
crc1 = pgFileGetCRC(path1, true, true, NULL, FIO_DB_HOST, false);
539539

540540
return EQ_CRC32C(crc1, crc2);
541541
}

‎src/backup.c

+4-4
Original file line numberDiff line numberDiff line change
@@ -555,7 +555,7 @@ do_backup_instance(PGconn *backup_conn)
555555
if (S_ISREG(file->mode))
556556
{
557557
file->crc = pgFileGetCRC(file->path, true, false,
558-
&file->read_size, FIO_BACKUP_HOST);
558+
&file->read_size, FIO_BACKUP_HOST, false);
559559
file->write_size = file->read_size;
560560
}
561561
/* Remove file path root prefix*/
@@ -1807,7 +1807,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn)
18071807
file = pgFileNew(backup_label, backup_label, true, 0,
18081808
FIO_BACKUP_HOST);
18091809
file->crc = pgFileGetCRC(file->path, true, false,
1810-
&file->read_size, FIO_BACKUP_HOST);
1810+
&file->read_size, FIO_BACKUP_HOST, false);
18111811
file->write_size = file->read_size;
18121812
free(file->path);
18131813
file->path = strdup(PG_BACKUP_LABEL_FILE);
@@ -1855,7 +1855,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn)
18551855
if (S_ISREG(file->mode))
18561856
{
18571857
file->crc = pgFileGetCRC(file->path, true, false,
1858-
&file->read_size, FIO_BACKUP_HOST);
1858+
&file->read_size, FIO_BACKUP_HOST, false);
18591859
file->write_size = file->read_size;
18601860
}
18611861
free(file->path);
@@ -2099,7 +2099,7 @@ backup_files(void *arg)
20992099
buf.st_mtime < current.parent_backup)
21002100
{
21012101
file->crc = pgFileGetCRC(file->path, true, false,
2102-
&file->read_size, FIO_DB_HOST);
2102+
&file->read_size, FIO_DB_HOST, false);
21032103
file->write_size = file->read_size;
21042104
/* ...and checksum is the same... */
21052105
if (EQ_TRADITIONAL_CRC32(file->crc, (*prev_file)->crc))

‎src/data.c

+18-20
Original file line numberDiff line numberDiff line change
@@ -727,7 +727,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate,
727727
if (file->write_size != BYTES_INVALID)
728728
{
729729
/* open backup mode file for read */
730-
in = fopen(file->path, PG_BINARY_R);
730+
in = fio_fopen(file->path, PG_BINARY_R, FIO_LOCAL_HOST, file->is_datafile && instance_config.encryption);
731731
if (in == NULL)
732732
{
733733
elog(ERROR, "Cannot open backup file \"%s\": %s", file->path,
@@ -744,15 +744,15 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate,
744744
if (out == NULL)
745745
{
746746
int errno_tmp = errno;
747-
fclose(in);
747+
fio_fclose(in);
748748
elog(ERROR, "Cannot open restore target file \"%s\": %s",
749749
to_path, strerror(errno_tmp));
750750
}
751751

752752
while (true)
753753
{
754754
off_t write_pos;
755-
size_t read_len;
755+
ssize_t read_len;
756756
DataPage compressed_page; /* used as read buffer */
757757
DataPage page;
758758
int32 uncompressed_size = 0;
@@ -777,13 +777,13 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate,
777777
}
778778

779779
/* read BackupPageHeader */
780-
read_len = fread(&header, 1, sizeof(header), in);
780+
read_len = fio_fread(in, &header, sizeof(header));
781781
if (read_len != sizeof(header))
782782
{
783783
int errno_tmp = errno;
784-
if (read_len == 0 && feof(in))
784+
if (read_len == 0)
785785
break; /* EOF found */
786-
else if (read_len != 0 && feof(in))
786+
else if (read_len > 0)
787787
elog(ERROR,
788788
"Odd size page found at block %u of \"%s\"",
789789
blknum, file->path);
@@ -818,8 +818,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate,
818818
Assert(header.compressed_size <= BLCKSZ);
819819

820820
/* read a page from file */
821-
read_len = fread(compressed_page.data, 1,
822-
MAXALIGN(header.compressed_size), in);
821+
read_len = fio_fread(in, compressed_page.data, MAXALIGN(header.compressed_size));
823822
if (read_len != MAXALIGN(header.compressed_size))
824823
elog(ERROR, "Cannot read block %u of \"%s\" read %zu of %d",
825824
blknum, file->path, read_len, header.compressed_size);
@@ -926,7 +925,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate,
926925
int errno_tmp = errno;
927926

928927
if (in)
929-
fclose(in);
928+
fio_fclose(in);
930929
fio_fclose(out);
931930
elog(ERROR, "Cannot change mode of \"%s\": %s", to_path,
932931
strerror(errno_tmp));
@@ -937,7 +936,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate,
937936
elog(ERROR, "Cannot write \"%s\": %s", to_path, strerror(errno));
938937

939938
if (in)
940-
fclose(in);
939+
fio_fclose(in);
941940
}
942941

943942
/*
@@ -964,7 +963,7 @@ copy_file(fio_location from_location, const char *to_root,
964963
file->write_size = 0;
965964

966965
/* open backup mode file for read */
967-
in = fio_fopen(file->path, PG_BINARY_R, from_location, instance_config.encryption && from_location == FIO_BACKUP_HOST);
966+
in = fio_fopen(file->path, PG_BINARY_R, from_location, file->is_datafile && instance_config.encryption && from_location == FIO_BACKUP_HOST);
968967
if (in == NULL)
969968
{
970969
FIN_FILE_CRC32(true, crc);
@@ -989,7 +988,7 @@ copy_file(fio_location from_location, const char *to_root,
989988

990989
/* open backup file for write */
991990
join_path_components(to_path, to_root, file->rel_path);
992-
out = fio_fopen(to_path, PG_BINARY_W, to_location, instance_config.encryption && to_location == FIO_BACKUP_HOST);
991+
out = fio_fopen(to_path, PG_BINARY_W, to_location, file->is_datafile && instance_config.encryption && to_location == FIO_BACKUP_HOST);
993992
if (out == NULL)
994993
{
995994
int errno_tmp = errno;
@@ -1265,15 +1264,15 @@ bool
12651264
check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
12661265
uint32 backup_version)
12671266
{
1268-
size_t read_len = 0;
1267+
ssize_t read_len = 0;
12691268
bool is_valid = true;
12701269
FILE *in;
12711270
pg_crc32 crc;
12721271
bool use_crc32c = backup_version <= 20021 || backup_version >= 20025;
12731272

12741273
elog(VERBOSE, "Validate relation blocks for file %s", file->path);
12751274

1276-
in = fopen(file->path, PG_BINARY_R);
1275+
in = fio_fopen(file->path, PG_BINARY_R, FIO_BACKUP_HOST, instance_config.encryption);
12771276
if (in == NULL)
12781277
{
12791278
if (errno == ENOENT)
@@ -1301,13 +1300,13 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
13011300
elog(ERROR, "Interrupted during data file validation");
13021301

13031302
/* read BackupPageHeader */
1304-
read_len = fread(&header, 1, sizeof(header), in);
1303+
read_len = fio_fread(in, &header, sizeof(header));
13051304
if (read_len != sizeof(header))
13061305
{
13071306
int errno_tmp = errno;
1308-
if (read_len == 0 && feof(in))
1307+
if (read_len == 0)
13091308
break; /* EOF found */
1310-
else if (read_len != 0 && feof(in))
1309+
else if (read_len > 0)
13111310
elog(WARNING,
13121311
"Odd size page found at block %u of \"%s\"",
13131312
blknum, file->path);
@@ -1343,8 +1342,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
13431342

13441343
Assert(header.compressed_size <= BLCKSZ);
13451344

1346-
read_len = fread(compressed_page.data, 1,
1347-
MAXALIGN(header.compressed_size), in);
1345+
read_len = fio_fread(in, compressed_page.data, MAXALIGN(header.compressed_size));
13481346
if (read_len != MAXALIGN(header.compressed_size))
13491347
{
13501348
elog(WARNING, "Cannot read block %u of \"%s\" read %zu of %d",
@@ -1395,7 +1393,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
13951393
}
13961394

13971395
FIN_FILE_CRC32(use_crc32c, crc);
1398-
fclose(in);
1396+
fio_fclose(in);
13991397

14001398
if (crc != file->crc)
14011399
{

‎src/dir.c

+2-2
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ pgFileDelete(pgFile *file)
251251

252252
pg_crc32
253253
pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted,
254-
size_t *bytes_read, fio_location location)
254+
size_t *bytes_read, fio_location location, bool encryption)
255255
{
256256
FILE *fp;
257257
pg_crc32 crc = 0;
@@ -263,7 +263,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted,
263263
INIT_FILE_CRC32(use_crc32c, crc);
264264

265265
/* open file in binary read mode */
266-
fp = fio_fopen(file_path, PG_BINARY_R, location, location == FIO_BACKUP_HOST && instance_config.encryption);
266+
fp = fio_fopen(file_path, PG_BINARY_R, location, encryption);
267267
if (fp == NULL)
268268
{
269269
if (!raise_on_deleted && errno == ENOENT)

‎src/help.c

+6
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ help_pg_probackup(void)
117117
printf(_(" [--delete-expired] [--delete-wal] [--merge-expired]\n"));
118118
printf(_(" [--retention-redundancy=retention-redundancy]\n"));
119119
printf(_(" [--retention-window=retention-window]\n"));
120+
printf(_(" [--encryption]\n"));
120121
printf(_(" [--compress]\n"));
121122
printf(_(" [--compress-algorithm=compress-algorithm]\n"));
122123
printf(_(" [--compress-level=compress-level]\n"));
@@ -182,6 +183,7 @@ help_pg_probackup(void)
182183
printf(_(" --wal-file-path=wal-file-path\n"));
183184
printf(_(" --wal-file-name=wal-file-name\n"));
184185
printf(_(" [--overwrite]\n"));
186+
printf(_(" [--encryption]\n"));
185187
printf(_(" [--compress]\n"));
186188
printf(_(" [--compress-algorithm=compress-algorithm]\n"));
187189
printf(_(" [--compress-level=compress-level]\n"));
@@ -233,6 +235,7 @@ help_backup(void)
233235
printf(_(" [--delete-expired] [--delete-wal] [--merge-expired]\n"));
234236
printf(_(" [--retention-redundancy=retention-redundancy]\n"));
235237
printf(_(" [--retention-window=retention-window]\n"));
238+
printf(_(" [--encryption]\n"));
236239
printf(_(" [--compress]\n"));
237240
printf(_(" [--compress-algorithm=compress-algorithm]\n"));
238241
printf(_(" [--compress-level=compress-level]\n"));
@@ -300,6 +303,9 @@ help_backup(void)
300303
printf(_(" --compress-level=compress-level\n"));
301304
printf(_(" level of compression [0-9] (default: 1)\n"));
302305

306+
printf(_("\n Encryption options:\n"));
307+
printf(_(" --encryption enable backup encryption using PG_CIPHER_KEY\n"));
308+
303309
printf(_("\n Archive options:\n"));
304310
printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n"));
305311

‎src/merge.c

+2-2
Original file line numberDiff line numberDiff line change
@@ -513,7 +513,7 @@ merge_files(void *arg)
513513
* Recalculate crc for backup prior to 2.0.25.
514514
*/
515515
if (parse_program_version(from_backup->program_version) < 20025)
516-
file->crc = pgFileGetCRC(to_file_path, true, true, NULL, FIO_LOCAL_HOST);
516+
file->crc = pgFileGetCRC(to_file_path, true, true, NULL, FIO_LOCAL_HOST, instance_config.encryption);
517517
/* Otherwise just get it from the previous file */
518518
else
519519
file->crc = to_file->crc;
@@ -637,7 +637,7 @@ merge_files(void *arg)
637637
* do that.
638638
*/
639639
file->write_size = pgFileSize(to_file_path);
640-
file->crc = pgFileGetCRC(to_file_path, true, true, NULL, FIO_LOCAL_HOST);
640+
file->crc = pgFileGetCRC(to_file_path, true, true, NULL, FIO_LOCAL_HOST, instance_config.encryption);
641641
}
642642
}
643643
else if (file->external_dir_num)

‎src/parsexlog.c

+2-2
Original file line numberDiff line numberDiff line change
@@ -736,7 +736,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
736736
return -1;
737737
}
738738
reader_data->xlogfile = fio_fdopen(reader_data->xlogpath, fd, "rb",
739-
instance_config.encryption);
739+
instance_config.encryption && !stream_wal);
740740
}
741741
#ifdef HAVE_LIBZ
742742
/* Try to open compressed WAL segment */
@@ -752,7 +752,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
752752
reader_data->xlogexists = true;
753753
reader_data->gz_xlogfile = fio_gzopen(reader_data->gz_xlogpath,
754754
"rb", -1, FIO_BACKUP_HOST,
755-
instance_config.encryption);
755+
instance_config.encryption && !stream_wal);
756756
if (reader_data->gz_xlogfile == NULL)
757757
{
758758
elog(WARNING, "Thread [%d]: Could not open compressed WAL segment \"%s\": %s",

‎src/pg_probackup.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ static ConfigOption cmd_options[] =
197197
/* show options */
198198
{ 'f', 153, "format", opt_show_format, SOURCE_CMD_STRICT },
199199
/* encryption options */
200-
{ 'b', 154, "encryption", &encryption_shortcut,SOURCE_CMD_STRICT },
200+
{ 'b', 158, "encryption", &encryption_shortcut,SOURCE_CMD_STRICT },
201201

202202
/* options for backward compatibility */
203203
{ 's', 136, "time", &target_time, SOURCE_CMD_STRICT },

‎src/pg_probackup.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -628,7 +628,7 @@ extern pgFile *pgFileInit(const char *path, const char *rel_path);
628628
extern void pgFileDelete(pgFile *file);
629629
extern void pgFileFree(void *file);
630630
extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c,
631-
bool raise_on_deleted, size_t *bytes_read, fio_location location);
631+
bool raise_on_deleted, size_t *bytes_read, fio_location location, bool encryption);
632632
extern int pgFileCompareName(const void *f1, const void *f2);
633633
extern int pgFileComparePath(const void *f1, const void *f2);
634634
extern int pgFileComparePathWithExternal(const void *f1, const void *f2);

0 commit comments

Comments
 (0)