Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 04def9f

Browse files
michael-grundermichael-grunder
authored
Rebased LZ4 PR (#1781)
LZ4 compression by @iliaal
1 parent 5ca4141 commit 04def9f

6 files changed

Lines changed: 208 additions & 1 deletion

File tree

‎common.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ typedef enum {
100100
#define REDIS_COMPRESSION_NONE 0
101101
#define REDIS_COMPRESSION_LZF 1
102102
#define REDIS_COMPRESSION_ZSTD 2
103+
#define REDIS_COMPRESSION_LZ4 3
103104

104105
/* SCAN options */
105106
#define REDIS_SCAN_NORETRY 0

‎config.m4‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,12 @@ PHP_ARG_ENABLE(redis-zstd, whether to enable Zstd compression,
2929
PHP_ARG_WITH(libzstd, use system libsztd,
3030
[ --with-libzstd[=DIR] Use system libzstd], yes, no)
3131

32+
PHP_ARG_ENABLE(redis-lz4, whether to enable lz4 compression,
33+
[ --enable-redis-lz4 Enable lz4 compression support], no, no)
34+
35+
PHP_ARG_WITH(liblz4, use system liblz4,
36+
[ --with-liblz4[=DIR] Use system liblz4], no, no)
37+
3238
if test "$PHP_REDIS" != "no"; then
3339

3440
if test "$PHP_REDIS_SESSION" != "no"; then
@@ -194,6 +200,31 @@ if test "$PHP_REDIS" != "no"; then
194200
fi
195201
fi
196202

203+
if test "$PHP_REDIS_LZ4" != "no"; then
204+
AC_DEFINE(HAVE_REDIS_LZ4, 1, [ ])
205+
AC_MSG_CHECKING(for liblz4 files in default path)
206+
for i in $PHP_LIBLZ4 /usr/local /usr; do
207+
if test -r $i/include/lz4.h; then
208+
AC_MSG_RESULT(found in $i)
209+
LIBLZ4_DIR=$i
210+
break
211+
fi
212+
done
213+
if test -z "$LIBLZ4_DIR"; then
214+
AC_MSG_RESULT([not found])
215+
AC_MSG_ERROR([Please reinstall the liblz4 distribution])
216+
fi
217+
PHP_CHECK_LIBRARY(lz4, LZ4_compress,
218+
[
219+
PHP_ADD_LIBRARY_WITH_PATH(zstd, $LIBLZ4_DIR/$PHP_LIBDIR, REDIS_SHARED_LIBADD)
220+
], [
221+
AC_MSG_ERROR([could not find usable liblz4])
222+
], [
223+
-L$LIBLZ4_DIR/$PHP_LIBDIR
224+
])
225+
PHP_SUBST(REDIS_SHARED_LIBADD)
226+
fi
227+
197228
if test "$PHP_REDIS_ZSTD" != "no"; then
198229
AC_DEFINE(HAVE_REDIS_ZSTD, 1, [ ])
199230
if test "$PHP_LIBZSTD" != "no"; then

‎library.c‎

Lines changed: 137 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,26 @@
2525
#include <zstd.h>
2626
#endif
2727

28+
#ifdef HAVE_REDIS_LZ4
29+
#include <lz4.h>
30+
#include <lz4hc.h>
31+
32+
/* uint8_t crf + int length */
33+
#define REDIS_LZ4_HDR_SIZE (sizeof(uint8_t) + sizeof(int))
34+
#if defined(LZ4HC_CLEVEL_MAX)
35+
/* version >= 1.7.5 */
36+
#define REDIS_LZ4_MAX_CLEVEL LZ4HC_CLEVEL_MAX
37+
38+
#elif defined (LZ4HC_MAX_CLEVEL)
39+
/* version >= 1.7.3 */
40+
#define REDIS_LZ4_MAX_CLEVEL LZ4HC_MAX_CLEVEL
41+
42+
#else
43+
/* older versions */
44+
#define REDIS_LZ4_MAX_CLEVEL 12
45+
#endif
46+
#endif
47+
2848
#include <zend_exceptions.h>
2949
#include "php_redis.h"
3050
#include "library.h"
@@ -2280,6 +2300,26 @@ PHP_REDIS_API void redis_free_socket(RedisSock *redis_sock)
22802300
efree(redis_sock);
22812301
}
22822302

2303+
#ifdef HAVE_REDIS_LZ4
2304+
/* Implementation of CRC8 for our LZ4 checksum value */
2305+
static uint8_t crc8(unsigned char *input, size_t len) {
2306+
size_t i;
2307+
uint8_t crc = 0xFF;
2308+
2309+
while (len--) {
2310+
crc ^= *input++;
2311+
for (i = 0; i < 8; i++) {
2312+
if (crc & 0x80)
2313+
crc = (uint8_t)(crc << 1) ^ 0x31;
2314+
else
2315+
crc <<= 1;
2316+
}
2317+
}
2318+
2319+
return crc;
2320+
}
2321+
#endif
2322+
22832323
PHP_REDIS_API int
22842324
redis_pack(RedisSock *redis_sock, zval *z, char **val, size_t *val_len)
22852325
{
@@ -2288,6 +2328,12 @@ redis_pack(RedisSock *redis_sock, zval *z, char **val, size_t *val_len)
22882328
size_t len;
22892329

22902330
valfree = redis_serialize(redis_sock, z, &buf, &len);
2331+
if (redis_sock->compression == REDIS_COMPRESSION_NONE) {
2332+
*val = buf;
2333+
*val_len = len;
2334+
return valfree;
2335+
}
2336+
22912337
switch (redis_sock->compression) {
22922338
case REDIS_COMPRESSION_LZF:
22932339
#ifdef HAVE_REDIS_LZF
@@ -2340,6 +2386,54 @@ redis_pack(RedisSock *redis_sock, zval *z, char **val, size_t *val_len)
23402386
}
23412387
efree(data);
23422388
}
2389+
#endif
2390+
break;
2391+
case REDIS_COMPRESSION_LZ4:
2392+
#ifdef HAVE_REDIS_LZ4
2393+
{
2394+
/* Compressing empty data is pointless */
2395+
if (len < 1)
2396+
break;
2397+
2398+
/* Compressing more than INT_MAX bytes would require multiple blocks */
2399+
if (len > INT_MAX) {
2400+
php_error_docref(NULL, E_WARNING,
2401+
"LZ4: compressing > %d bytes not supported", INT_MAX);
2402+
break;
2403+
}
2404+
2405+
int old_len = len, lz4len, lz4bound;
2406+
uint8_t crc = crc8((unsigned char*)&old_len, sizeof(old_len));
2407+
char *lz4buf, *lz4pos;
2408+
2409+
lz4bound = LZ4_compressBound(len);
2410+
lz4buf = emalloc(REDIS_LZ4_HDR_SIZE + lz4bound);
2411+
lz4pos = lz4buf;
2412+
2413+
/* Copy and move past crc8 length checksum */
2414+
memcpy(lz4pos, &crc, sizeof(crc));
2415+
lz4pos += sizeof(crc);
2416+
2417+
/* Copy and advance past length */
2418+
memcpy(lz4pos, &old_len, sizeof(old_len));
2419+
lz4pos += sizeof(old_len);
2420+
2421+
if (redis_sock->compression_level <= 0 || redis_sock->compression_level > REDIS_LZ4_MAX_CLEVEL) {
2422+
lz4len = LZ4_compress_default(buf, lz4pos, old_len, lz4bound);
2423+
} else {
2424+
lz4len = LZ4_compress_HC(buf, lz4pos, old_len, lz4bound, redis_sock->compression_level);
2425+
}
2426+
2427+
if (lz4len <= 0) {
2428+
efree(lz4buf);
2429+
break;
2430+
}
2431+
2432+
if (valfree) efree(buf);
2433+
*val = lz4buf;
2434+
*val_len = lz4len + REDIS_LZ4_HDR_SIZE;
2435+
return 1;
2436+
}
23432437
#endif
23442438
break;
23452439
}
@@ -2359,9 +2453,12 @@ redis_unpack(RedisSock *redis_sock, const char *val, int val_len, zval *z_ret)
23592453
int i;
23602454
uint32_t res;
23612455

2362-
errno = E2BIG;
2456+
if (val_len == 0)
2457+
break;
2458+
23632459
/* start from two-times bigger buffer and
23642460
* increase it exponentially if needed */
2461+
errno = E2BIG;
23652462
for (i = 2; errno == E2BIG; i *= 2) {
23662463
data = emalloc(i * val_len);
23672464
if ((res = lzf_decompress(val, val_len, data, i * val_len)) == 0) {
@@ -2397,6 +2494,45 @@ redis_unpack(RedisSock *redis_sock, const char *val, int val_len, zval *z_ret)
23972494
return 1;
23982495
}
23992496
}
2497+
#endif
2498+
break;
2499+
case REDIS_COMPRESSION_LZ4:
2500+
#ifdef HAVE_REDIS_LZ4
2501+
{
2502+
char *data;
2503+
int datalen;
2504+
uint8_t lz4crc;
2505+
2506+
/* We must have at least enough bytes for our header, and can't have more than
2507+
* INT_MAX + our header size. */
2508+
if (val_len < REDIS_LZ4_HDR_SIZE || val_len > INT_MAX + REDIS_LZ4_HDR_SIZE)
2509+
break;
2510+
2511+
/* Operate on copies in case our CRC fails */
2512+
const char *copy = val;
2513+
size_t copylen = val_len;
2514+
2515+
/* Read in our header bytes */
2516+
memcpy(&lz4crc, copy, sizeof(uint8_t));
2517+
copy += sizeof(uint8_t); copylen -= sizeof(uint8_t);
2518+
memcpy(&datalen, copy, sizeof(int));
2519+
copy += sizeof(int); copylen -= sizeof(int);
2520+
2521+
/* Make sure our CRC matches (TODO: Maybe issue a docref error?) */
2522+
if (crc8((unsigned char*)&datalen, sizeof(datalen)) != lz4crc)
2523+
break;
2524+
2525+
/* Finally attempt decompression */
2526+
data = emalloc(datalen);
2527+
if (LZ4_decompress_safe(copy, data, copylen, datalen) > 0) {
2528+
if (redis_unserialize(redis_sock, data, datalen, z_ret) == 0) {
2529+
ZVAL_STRINGL(z_ret, data, datalen);
2530+
}
2531+
efree(data);
2532+
return 1;
2533+
}
2534+
efree(data);
2535+
}
24002536
#endif
24012537
break;
24022538
}

‎redis.c‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@
4040
#include <zstd.h>
4141
#endif
4242

43+
#ifdef HAVE_REDIS_LZ4
44+
#include <lz4.h>
45+
#endif
46+
4347
#ifdef PHP_SESSION
4448
extern ps_module ps_mod_redis;
4549
extern ps_module ps_mod_redis_cluster;
@@ -719,6 +723,10 @@ static void add_class_constants(zend_class_entry *ce, int is_cluster) {
719723
zend_declare_class_constant_long(ce, ZEND_STRL("COMPRESSION_ZSTD_MAX"), ZSTD_maxCLevel());
720724
#endif
721725

726+
#ifdef HAVE_REDIS_LZ4
727+
zend_declare_class_constant_long(ce, ZEND_STRL("COMPRESSION_LZ4"), REDIS_COMPRESSION_LZ4);
728+
#endif
729+
722730
/* scan options*/
723731
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_SCAN"), REDIS_OPT_SCAN);
724732
zend_declare_class_constant_long(ce, ZEND_STRL("SCAN_RETRY"), REDIS_SCAN_RETRY);
@@ -886,6 +894,12 @@ PHP_MINFO_FUNCTION(redis)
886894
smart_str_appends(&names, ", ");
887895
}
888896
smart_str_appends(&names, "zstd");
897+
#endif
898+
#ifdef HAVE_REDIS_LZ4
899+
if (names.s) {
900+
smart_str_appends(&names, ", ");
901+
}
902+
smart_str_appends(&names, "lz4");
889903
#endif
890904
if (names.s) {
891905
smart_str_0(&names);

‎redis_commands.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4002,6 +4002,9 @@ void redis_setoption_handler(INTERNAL_FUNCTION_PARAMETERS,
40024002
#endif
40034003
#ifdef HAVE_REDIS_ZSTD
40044004
|| val_long == REDIS_COMPRESSION_ZSTD
4005+
#endif
4006+
#ifdef HAVE_REDIS_LZ4
4007+
|| val_long == REDIS_COMPRESSION_LZ4
40054008
#endif
40064009
) {
40074010
redis_sock->compression = val_long;

‎tests/RedisTest.php‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4519,6 +4519,16 @@ public function testCompressionZSTD()
45194519
$this->checkCompression(Redis::COMPRESSION_ZSTD, 9);
45204520
}
45214521

4522+
4523+
public function testCompressionLZ4()
4524+
{
4525+
if (!defined('Redis::COMPRESSION_LZ4')) {
4526+
$this->markTestSkipped();
4527+
}
4528+
$this->checkCompression(Redis::COMPRESSION_LZ4, 0);
4529+
$this->checkCompression(Redis::COMPRESSION_LZ4, 9);
4530+
}
4531+
45224532
private function checkCompression($mode, $level)
45234533
{
45244534
$this->assertTrue($this->redis->setOption(Redis::OPT_COMPRESSION, $mode) === TRUE); // set ok
@@ -4530,6 +4540,18 @@ private function checkCompression($mode, $level)
45304540
$val = 'xxxxxxxxxx';
45314541
$this->redis->set('key', $val);
45324542
$this->assertEquals($val, $this->redis->get('key'));
4543+
4544+
/* Empty data */
4545+
$this->redis->set('key', '');
4546+
$this->assertEquals('', $this->redis->get('key'));
4547+
4548+
/* Iterate through class sizes */
4549+
for ($i = 1; $i <= 65536; $i *= 2) {
4550+
foreach ([str_repeat('A', $i), random_bytes($i)] as $val) {
4551+
$this->redis->set('key', $val);
4552+
$this->assertEquals($val, $this->redis->get('key'));
4553+
}
4554+
}
45334555
}
45344556

45354557
public function testDumpRestore() {

0 commit comments

Comments
 (0)