Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit ab6b7b1

Browse files
knizhnikknizhnik
committed
Add support of lz4 compression
1 parent d4da7e8 commit ab6b7b1

File tree

12 files changed

+438
-181
lines changed

12 files changed

+438
-181
lines changed

‎Makefile

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,14 @@ EXTRA_CLEAN = src/pg_crc.c src/datapagemap.c src/datapagemap.h \
1919

2020
INCLUDES = src/datapagemap.h src/streamutil.h src/receivelog.h
2121

22+
ifdef HAVE_LZ4
23+
COMPRESS_FLAGS += -DHAVE_LZ4
24+
COMPRESS_LIBS += -llz4
25+
endif
26+
27+
PG_LIBS = $(COMPRESS_LIBS)
28+
PG_CPPFLAGS = $(COMPRESS_FLAGS)
29+
2230
ifdef USE_PGXS
2331
PG_CONFIG = pg_config
2432
PGXS := $(shell $(PG_CONFIG) --pgxs)
@@ -50,9 +58,8 @@ EXTRA_CLEAN += src/walmethods.c src/walmethods.h
5058
INCLUDES += src/walmethods.h
5159
endif
5260

53-
5461
PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_srcdir)/$(subdir)/src
55-
override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS)
62+
override CPPFLAGS := -DFRONTEND -DHAVE_LZ4 $(CPPFLAGS) $(PG_CPPFLAGS)
5663
PG_LIBS_INTERNAL = $(libpq_pgport) ${PTHREAD_CFLAGS}
5764

5865
all: checksrcdir $(INCLUDES);

‎src/archive.c

Lines changed: 46 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,11 @@
1313
#include <unistd.h>
1414

1515
static void push_wal_file(const char *from_path, const char *to_path,
16-
bool is_compress, bool overwrite, int compress_level);
16+
CompressAlg compress_alg, bool overwrite, int compress_level);
1717
static void get_wal_file(const char *from_path, const char *to_path);
18-
#ifdef HAVE_LIBZ
18+
1919
static const char *get_gz_error(gzFile gzf, int errnum);
20-
#endif
21-
static bool fileEqualCRC(const char *path1, const char *path2,
22-
bool path2_is_compressed);
20+
static bool fileEqualCRC(const char *path1, const char *path2, CompressAlg compress_alg);
2321
static void copy_file_attributes(const char *from_path,
2422
fio_location from_location,
2523
const char *to_path, fio_location to_location,
@@ -40,7 +38,7 @@ do_archive_push(InstanceConfig *instance,
4038
char absolute_wal_file_path[MAXPGPATH];
4139
char current_dir[MAXPGPATH];
4240
uint64 system_id;
43-
bool is_compress = false;
41+
CompressAlg compress_alg = NONE_COMPRESS;
4442

4543
if (wal_file_name == NULL && wal_file_path == NULL)
4644
elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p");
@@ -80,11 +78,15 @@ do_archive_push(InstanceConfig *instance,
8078
elog(ERROR, "pglz compression is not supported");
8179

8280
#ifdef HAVE_LIBZ
83-
if (instance->compress_alg == ZLIB_COMPRESS)
84-
is_compress = IsXLogFileName(wal_file_name);
81+
if (instance->compress_alg == ZLIB_COMPRESS && IsXLogFileName(wal_file_name))
82+
compress_alg = ZLIB_COMPRESS;
83+
#endif
84+
#ifdef HAVE_LZ4
85+
if (instance->compress_alg == LZ4_COMPRESS && IsXLogFileName(wal_file_name))
86+
compress_alg = LZ4_COMPRESS;
8587
#endif
8688

87-
push_wal_file(absolute_wal_file_path, backup_wal_file_path, is_compress,
89+
push_wal_file(absolute_wal_file_path, backup_wal_file_path, compress_alg,
8890
overwrite, instance->compress_level);
8991
elog(INFO, "pg_probackup archive-push completed successfully");
9092

@@ -133,7 +135,7 @@ do_archive_get(InstanceConfig *instance,
133135
* Copy WAL segment from pgdata to archive catalog with possible compression.
134136
*/
135137
void
136-
push_wal_file(const char *from_path, const char *to_path, bool is_compress,
138+
push_wal_file(const char *from_path, const char *to_path, CompressAlg compress_alg,
137139
bool overwrite, int compress_level)
138140
{
139141
FILE *in = NULL;
@@ -148,17 +150,17 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress,
148150
int partial_file_size = 0;
149151
bool partial_file_exists = false;
150152

151-
#ifdef HAVE_LIBZ
152153
char gz_to_path[MAXPGPATH];
153154
gzFile gz_out = NULL;
154155

155-
if (is_compress)
156+
if (compress_alg != NONE_COMPRESS)
156157
{
157-
snprintf(gz_to_path, sizeof(gz_to_path), "%s.gz", to_path);
158+
Assert(compress_alg == ZLIB_COMPRESS || compress_alg == LZ4_COMPRESS);
159+
snprintf(gz_to_path, sizeof(gz_to_path), "%s.%s",
160+
to_path, compress_alg == ZLIB_COMPRESS ? "gz" : "lz4");
158161
to_path_p = gz_to_path;
159162
}
160163
else
161-
#endif
162164
to_path_p = to_path;
163165

164166
/* open file for read */
@@ -170,20 +172,19 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress,
170172
/* Check if possible to skip copying */
171173
if (fileExists(to_path_p, FIO_BACKUP_HOST))
172174
{
173-
if (fileEqualCRC(from_path, to_path_p, is_compress))
175+
if (fileEqualCRC(from_path, to_path_p, compress_alg))
174176
return;
175177
/* Do not copy and do not rise error. Just quit as normal. */
176178
else if (!overwrite)
177179
elog(ERROR, "WAL segment \"%s\" already exists.", to_path_p);
178180
}
179181

180182
/* open backup file for write */
181-
#ifdef HAVE_LIBZ
182-
if (is_compress)
183+
if (compress_alg != NONE_COMPRESS)
183184
{
184185
snprintf(to_path_temp, sizeof(to_path_temp), "%s.part", gz_to_path);
185186

186-
gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, compress_level, FIO_BACKUP_HOST);
187+
gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, compress_level, FIO_BACKUP_HOST, compress_alg);
187188
if (gz_out == NULL)
188189
{
189190
partial_file_exists = true;
@@ -192,7 +193,6 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress,
192193
}
193194
}
194195
else
195-
#endif
196196
{
197197
snprintf(to_path_temp, sizeof(to_path_temp), "%s.part", to_path);
198198

@@ -243,16 +243,14 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress,
243243
elog(WARNING, "Reusing stale destination temporary WAL file \"%s\"", to_path_temp);
244244
fio_unlink(to_path_temp, FIO_BACKUP_HOST);
245245

246-
#ifdef HAVE_LIBZ
247-
if (is_compress)
246+
if (compress_alg != NONE_COMPRESS)
248247
{
249-
gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, compress_level, FIO_BACKUP_HOST);
248+
gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, compress_level, FIO_BACKUP_HOST, compress_alg);
250249
if (gz_out == NULL)
251250
elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s",
252251
to_path_temp, strerror(errno));
253252
}
254253
else
255-
#endif
256254
{
257255
out = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST);
258256
if (out < 0)
@@ -279,8 +277,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress,
279277

280278
if (read_len > 0)
281279
{
282-
#ifdef HAVE_LIBZ
283-
if (is_compress)
280+
if (compress_alg != NONE_COMPRESS)
284281
{
285282
if (fio_gzwrite(gz_out, buf, read_len) != read_len)
286283
{
@@ -291,7 +288,6 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress,
291288
}
292289
}
293290
else
294-
#endif
295291
{
296292
if (fio_write(out, buf, read_len) != read_len)
297293
{
@@ -307,8 +303,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress,
307303
break;
308304
}
309305

310-
#ifdef HAVE_LIBZ
311-
if (is_compress)
306+
if (compress_alg != NONE_COMPRESS)
312307
{
313308
if (fio_gzclose(gz_out) != 0)
314309
{
@@ -319,7 +314,6 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress,
319314
}
320315
}
321316
else
322-
#endif
323317
{
324318
if (fio_flush(out) != 0 || fio_close(out) != 0)
325319
{
@@ -349,10 +343,8 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress,
349343
to_path_temp, to_path_p, strerror(errno_temp));
350344
}
351345

352-
#ifdef HAVE_LIBZ
353-
if (is_compress)
346+
if (compress_alg != NONE_COMPRESS)
354347
elog(INFO, "WAL file compressed to \"%s\"", gz_to_path);
355-
#endif
356348
}
357349

358350
/*
@@ -367,17 +359,13 @@ get_wal_file(const char *from_path, const char *to_path)
367359
const char *from_path_p = from_path;
368360
char to_path_temp[MAXPGPATH];
369361
int errno_temp;
370-
bool is_decompress = false;
371-
372-
#ifdef HAVE_LIBZ
362+
CompressAlg compress_alg = NONE_COMPRESS;
373363
char gz_from_path[MAXPGPATH];
374364
gzFile gz_in = NULL;
375-
#endif
376365

377366
/* First check source file for existance */
378367
if (fio_access(from_path, F_OK, FIO_BACKUP_HOST) != 0)
379368
{
380-
#ifdef HAVE_LIBZ
381369
/*
382370
* Maybe we need to decompress the file. Check it with .gz
383371
* extension.
@@ -386,34 +374,41 @@ get_wal_file(const char *from_path, const char *to_path)
386374
if (fio_access(gz_from_path, F_OK, FIO_BACKUP_HOST) == 0)
387375
{
388376
/* Found compressed file */
389-
is_decompress = true;
377+
compress_alg = ZLIB_COMPRESS;
390378
from_path_p = gz_from_path;
391379
}
392-
#endif
380+
else
381+
{
382+
snprintf(gz_from_path, sizeof(gz_from_path), "%s.lz4", from_path);
383+
if (fio_access(gz_from_path, F_OK, FIO_BACKUP_HOST) == 0)
384+
{
385+
/* Found compressed file */
386+
compress_alg = LZ4_COMPRESS;
387+
from_path_p = gz_from_path;
388+
}
389+
}
393390
/* Didn't find compressed file */
394-
if (!is_decompress)
391+
if (compress_alg == NONE_COMPRESS)
395392
elog(ERROR, "Source WAL file \"%s\" doesn't exist",
396393
from_path);
397394
}
398395

399396
/* open file for read */
400-
if (!is_decompress)
397+
if (compress_alg == NONE_COMPRESS)
401398
{
402399
in = fio_fopen(from_path, PG_BINARY_R, FIO_BACKUP_HOST);
403400
if (in == NULL)
404401
elog(ERROR, "Cannot open source WAL file \"%s\": %s",
405402
from_path, strerror(errno));
406403
}
407-
#ifdef HAVE_LIBZ
408404
else
409405
{
410406
gz_in = fio_gzopen(gz_from_path, PG_BINARY_R, Z_DEFAULT_COMPRESSION,
411-
FIO_BACKUP_HOST);
407+
FIO_BACKUP_HOST, compress_alg);
412408
if (gz_in == NULL)
413409
elog(ERROR, "Cannot open compressed WAL file \"%s\": %s",
414410
gz_from_path, strerror(errno));
415411
}
416-
#endif
417412

418413
/* open backup file for write */
419414
snprintf(to_path_temp, sizeof(to_path_temp), "%s.part", to_path);
@@ -428,8 +423,7 @@ get_wal_file(const char *from_path, const char *to_path)
428423
{
429424
int read_len = 0;
430425

431-
#ifdef HAVE_LIBZ
432-
if (is_decompress)
426+
if (compress_alg != NONE_COMPRESS)
433427
{
434428
read_len = fio_gzread(gz_in, buf, sizeof(buf));
435429
if (read_len <= 0 && !fio_gzeof(gz_in))
@@ -441,7 +435,6 @@ get_wal_file(const char *from_path, const char *to_path)
441435
}
442436
}
443437
else
444-
#endif
445438
{
446439
read_len = fio_fread(in, buf, sizeof(buf));
447440
if (read_len < 0)
@@ -465,14 +458,12 @@ get_wal_file(const char *from_path, const char *to_path)
465458
}
466459

467460
/* Check for EOF */
468-
#ifdef HAVE_LIBZ
469-
if (is_decompress)
461+
if (compress_alg != NONE_COMPRESS)
470462
{
471463
if (fio_gzeof(gz_in) || read_len == 0)
472464
break;
473465
}
474466
else
475-
#endif
476467
{
477468
if (/* feof(in) || */ read_len == 0)
478469
break;
@@ -487,8 +478,7 @@ get_wal_file(const char *from_path, const char *to_path)
487478
to_path_temp, strerror(errno_temp));
488479
}
489480

490-
#ifdef HAVE_LIBZ
491-
if (is_decompress)
481+
if (compress_alg != NONE_COMPRESS)
492482
{
493483
if (fio_gzclose(gz_in) != 0)
494484
{
@@ -499,7 +489,6 @@ get_wal_file(const char *from_path, const char *to_path)
499489
}
500490
}
501491
else
502-
#endif
503492
{
504493
if (fio_fclose(in))
505494
{
@@ -521,13 +510,10 @@ get_wal_file(const char *from_path, const char *to_path)
521510
to_path_temp, to_path, strerror(errno_temp));
522511
}
523512

524-
#ifdef HAVE_LIBZ
525-
if (is_decompress)
513+
if (compress_alg != NONE_COMPRESS)
526514
elog(INFO, "WAL file decompressed from \"%s\"", gz_from_path);
527-
#endif
528515
}
529516

530-
#ifdef HAVE_LIBZ
531517
/*
532518
* Show error during work with compressed file
533519
*/
@@ -543,27 +529,25 @@ get_gz_error(gzFile gzf, int errnum)
543529
else
544530
return errmsg;
545531
}
546-
#endif
547532

548533
/*
549534
* compare CRC of two WAL files.
550535
* If necessary, decompress WAL file from path2
551536
*/
552537
static bool
553-
fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed)
538+
fileEqualCRC(const char *path1, const char *path2, CompressAlg compress_alg)
554539
{
555540
pg_crc32 crc1;
556541
pg_crc32 crc2;
557542

558543
/* Get checksum of backup file */
559-
#ifdef HAVE_LIBZ
560-
if (path2_is_compressed)
544+
if (compress_alg != NONE_COMPRESS)
561545
{
562546
char buf [1024];
563547
gzFile gz_in = NULL;
564548

565549
INIT_FILE_CRC32(true, crc2);
566-
gz_in = fio_gzopen(path2, PG_BINARY_R, Z_DEFAULT_COMPRESSION, FIO_BACKUP_HOST);
550+
gz_in = fio_gzopen(path2, PG_BINARY_R, Z_DEFAULT_COMPRESSION, FIO_BACKUP_HOST, compress_alg);
567551
if (gz_in == NULL)
568552
/* File cannot be read */
569553
elog(ERROR,
@@ -592,7 +576,6 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed)
592576
path2, get_gz_error(gz_in, errno));
593577
}
594578
else
595-
#endif
596579
{
597580
crc2 = pgFileGetCRC(path2, true, true, NULL, FIO_BACKUP_HOST);
598581
}

0 commit comments

Comments
 (0)