Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 8cb2d5b

Browse files
yatsukhnenkoyatsukhnenko
committed
Issue #1048
This commit is adding support of data compression. LZF was choosen because it small and fast and Redis server uses it. Since [pecl package](https://pecl.php.net/package/lzf) doesn't provide lzf.h file after installing, LZF library was added as submodule. Another algorythms may be easely added by analogy with serializers. TODO: unit-tests for different data types.
1 parent b9ca16c commit 8cb2d5b

13 files changed

Lines changed: 151 additions & 27 deletions

‎.gitmodules‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[submodule "liblzf"]
2+
path = liblzf
3+
url = https://github.com/nemequ/liblzf.git

‎.travis.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,11 @@ addons:
3030
before_install:
3131
- phpize
3232
- pecl install igbinary
33-
- ./configure --enable-redis-igbinary CFLAGS=-Wall
33+
- ./configure --enable-redis-igbinary --enable-redis-lzf CFLAGS=-Wall
3434
install: make install
3535
before_script:
3636
- gem install redis
37-
- mkdir -p tests/nodes/ && echo >> tests/nodes/nodemap
37+
- mkdir -p tests/nodes/ && echo > tests/nodes/nodemap
3838
- for PORT in $(seq 6379 6382); do redis-server --port $PORT --daemonize yes; done
3939
- for PORT in $(seq 7000 7011); do redis-server --port $PORT --cluster-enabled yes --cluster-config-file $PORT.conf --daemonize yes; echo 127.0.0.1:$PORT >> tests/nodes/nodemap; done
4040
- wget https://raw.githubusercontent.com/antirez/redis/unstable/src/redis-trib.rb

‎cluster_library.c‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -456,7 +456,7 @@ void cluster_dist_add_val(redisCluster *c, clusterKeyVal *kv, zval *z_val
456456
int val_free;
457457

458458
// Serialize our value
459-
val_free = redis_serialize(c->flags, z_val, &val, &val_len TSRMLS_CC);
459+
val_free = redis_pack(c->flags, z_val, &val, &val_len TSRMLS_CC);
460460

461461
// Attach it to the provied keyval entry
462462
kv->val = val;
@@ -1495,12 +1495,12 @@ PHP_REDIS_API void cluster_bulk_resp(INTERNAL_FUNCTION_PARAMETERS, redisCluster
14951495
}
14961496

14971497
if (CLUSTER_IS_ATOMIC(c)) {
1498-
if (!redis_unserialize(c->flags, resp, c->reply_len, return_value TSRMLS_CC)) {
1498+
if (!redis_unpack(c->flags, resp, c->reply_len, return_value TSRMLS_CC)) {
14991499
CLUSTER_RETURN_STRING(c, resp, c->reply_len);
15001500
}
15011501
} else {
15021502
zval zv, *z = &zv;
1503-
if (redis_unserialize(c->flags, resp, c->reply_len, z TSRMLS_CC)) {
1503+
if (redis_unpack(c->flags, resp, c->reply_len, z TSRMLS_CC)) {
15041504
#if (PHP_MAJOR_VERSION < 7)
15051505
MAKE_STD_ZVAL(z);
15061506
*z = zv;
@@ -2322,7 +2322,7 @@ int mbulk_resp_loop(RedisSock *redis_sock, zval *z_result,
23222322

23232323
if (line != NULL) {
23242324
zval zv, *z = &zv;
2325-
if (redis_unserialize(redis_sock, line, line_len, z TSRMLS_CC)) {
2325+
if (redis_unpack(redis_sock, line, line_len, z TSRMLS_CC)) {
23262326
#if (PHP_MAJOR_VERSION < 7)
23272327
MAKE_STD_ZVAL(z);
23282328
*z = zv;
@@ -2367,7 +2367,7 @@ int mbulk_resp_loop_zipstr(RedisSock *redis_sock, zval *z_result,
23672367
} else {
23682368
/* Attempt serialization */
23692369
zval zv, *z = &zv;
2370-
if (redis_unserialize(redis_sock, line, line_len, z TSRMLS_CC)) {
2370+
if (redis_unpack(redis_sock, line, line_len, z TSRMLS_CC)) {
23712371
#if (PHP_MAJOR_VERSION < 7)
23722372
MAKE_STD_ZVAL(z);
23732373
*z = zv;
@@ -2406,7 +2406,7 @@ int mbulk_resp_loop_zipdbl(RedisSock *redis_sock, zval *z_result,
24062406
key_len = line_len;
24072407
} else {
24082408
zval zv, *z = &zv;
2409-
if (redis_unserialize(redis_sock,key,key_len, z TSRMLS_CC)) {
2409+
if (redis_unpack(redis_sock,key,key_len, z TSRMLS_CC)) {
24102410
zend_string *zstr = zval_get_string(z);
24112411
add_assoc_double_ex(z_result, ZSTR_VAL(zstr), ZSTR_LEN(zstr), atof(line));
24122412
zend_string_release(zstr);
@@ -2440,7 +2440,7 @@ int mbulk_resp_loop_assoc(RedisSock *redis_sock, zval *z_result,
24402440

24412441
if(line != NULL) {
24422442
zval zv, *z = &zv;
2443-
if (redis_unserialize(redis_sock, line, line_len, z TSRMLS_CC)) {
2443+
if (redis_unpack(redis_sock, line, line_len, z TSRMLS_CC)) {
24442444
#if (PHP_MAJOR_VERSION < 7)
24452445
MAKE_STD_ZVAL(z);
24462446
*z = zv;

‎common.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -478,6 +478,7 @@ typedef enum _PUBSUB_TYPE {
478478
#define REDIS_OPT_READ_TIMEOUT 3
479479
#define REDIS_OPT_SCAN 4
480480
#define REDIS_OPT_FAILOVER 5
481+
#define REDIS_OPT_COMPRESSION 6
481482

482483
/* cluster options */
483484
#define REDIS_FAILOVER_NONE 0
@@ -488,6 +489,9 @@ typedef enum _PUBSUB_TYPE {
488489
#define REDIS_SERIALIZER_NONE 0
489490
#define REDIS_SERIALIZER_PHP 1
490491
#define REDIS_SERIALIZER_IGBINARY 2
492+
/* compression */
493+
#define REDIS_COMPRESSION_NONE 0
494+
#define REDIS_COMPRESSION_LZF 1
491495

492496
/* SCAN options */
493497
#define REDIS_SCAN_NORETRY 0
@@ -651,6 +655,7 @@ typedef struct {
651655
zend_string *persistent_id;
652656

653657
int serializer;
658+
int compression;
654659
long dbNumber;
655660

656661
zend_string *prefix;

‎config.m4‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,16 @@ dnl config.m4 for extension redis
33

44
PHP_ARG_ENABLE(redis, whether to enable redis support,
55
dnl Make sure that the comment is aligned:
6-
[ --enable-redis Enable redis support])
6+
[ --enable-redis Enable redis support])
77

88
PHP_ARG_ENABLE(redis-session, whether to enable sessions,
9-
[ --disable-redis-session Disable session support], yes, no)
9+
[ --disable-redis-session Disable session support], yes, no)
1010

1111
PHP_ARG_ENABLE(redis-igbinary, whether to enable igbinary serializer support,
12-
[ --enable-redis-igbinary Enable igbinary serializer support], no, no)
12+
[ --enable-redis-igbinary Enable igbinary serializer support], no, no)
1313

14+
PHP_ARG_ENABLE(redis-lzf, whether to enable lzf compression,
15+
[ --enable-redis-lzf Enable lzf compression support], no, no)
1416

1517
if test "$PHP_REDIS" != "no"; then
1618

@@ -60,6 +62,13 @@ dnl Check for igbinary
6062
AC_MSG_RESULT([disabled])
6163
fi
6264

65+
if test "$PHP_REDIS_LZF" != "no"; then
66+
PHP_ADD_INCLUDE(liblzf)
67+
PHP_ADD_BUILD_DIR(liblzf)
68+
lzf_sources="liblzf/lzf_c.c liblzf/lzf_d.c"
69+
AC_DEFINE(HAVE_REDIS_LZF, 1, [ ])
70+
fi
71+
6372
dnl # --with-redis -> check with-path
6473
dnl SEARCH_PATH="/usr/local /usr" # you might want to change this
6574
dnl SEARCH_FOR="/include/redis.h" # you most likely want to change this
@@ -99,5 +108,5 @@ dnl Check for igbinary
99108
dnl
100109
dnl PHP_SUBST(REDIS_SHARED_LIBADD)
101110

102-
PHP_NEW_EXTENSION(redis, redis.c redis_commands.c library.c redis_session.c redis_array.c redis_array_impl.c redis_cluster.c cluster_library.c, $ext_shared)
111+
PHP_NEW_EXTENSION(redis, redis.c redis_commands.c library.c redis_session.c redis_array.c redis_array_impl.c redis_cluster.c cluster_library.c $lzf_sources, $ext_shared)
103112
fi

‎liblzf‎

Submodule liblzf added at fb25820

‎library.c‎

Lines changed: 67 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@
1111
#ifdef HAVE_REDIS_IGBINARY
1212
#include "igbinary/igbinary.h"
1313
#endif
14+
#ifdef HAVE_REDIS_LZF
15+
#include "lzf.h"
16+
#endif
1417
#include <zend_exceptions.h>
1518
#include "php_redis.h"
1619
#include "library.h"
@@ -602,7 +605,7 @@ redis_spprintf(RedisSock *redis_sock, short *slot TSRMLS_DC, char **ret, char *k
602605
break;
603606
case 'v':
604607
arg.zv = va_arg(ap, zval*);
605-
argfree = redis_serialize(redis_sock, arg.zv, &dup, &arglen TSRMLS_CC);
608+
argfree = redis_pack(redis_sock, arg.zv, &dup, &arglen TSRMLS_CC);
606609
redis_cmd_append_sstr(&cmd, dup, arglen);
607610
if (argfree) efree(dup);
608611
break;
@@ -710,7 +713,7 @@ int redis_cmd_append_sstr_zval(smart_string *str, zval *z, RedisSock *redis_sock
710713
strlen_t vallen;
711714
int valfree, retval;
712715

713-
valfree = redis_serialize(redis_sock, z, &val, &vallen TSRMLS_CC);
716+
valfree = redis_pack(redis_sock, z, &val, &vallen TSRMLS_CC);
714717
retval = redis_cmd_append_sstr(str, val, vallen);
715718
if (valfree) efree(val);
716719

@@ -1248,7 +1251,7 @@ PHP_REDIS_API void redis_string_response(INTERNAL_FUNCTION_PARAMETERS, RedisSock
12481251
}
12491252
IF_NOT_ATOMIC() {
12501253
zval zv, *z = &zv;
1251-
if (redis_unserialize(redis_sock, response, response_len, z TSRMLS_CC)) {
1254+
if (redis_unpack(redis_sock, response, response_len, z TSRMLS_CC)) {
12521255
#if (PHP_MAJOR_VERSION < 7)
12531256
MAKE_STD_ZVAL(z);
12541257
*z = zv;
@@ -1258,7 +1261,7 @@ PHP_REDIS_API void redis_string_response(INTERNAL_FUNCTION_PARAMETERS, RedisSock
12581261
add_next_index_stringl(z_tab, response, response_len);
12591262
}
12601263
} else {
1261-
if (!redis_unserialize(redis_sock, response, response_len, return_value TSRMLS_CC)) {
1264+
if (!redis_unpack(redis_sock, response, response_len, return_value TSRMLS_CC)) {
12621265
RETVAL_STRINGL(response, response_len);
12631266
}
12641267
}
@@ -1386,6 +1389,7 @@ redis_sock_create(char *host, int host_len, unsigned short port,
13861389
redis_sock->read_timeout = read_timeout;
13871390

13881391
redis_sock->serializer = REDIS_SERIALIZER_NONE;
1392+
redis_sock->compression = REDIS_COMPRESSION_NONE;
13891393
redis_sock->mode = ATOMIC;
13901394
redis_sock->head = NULL;
13911395
redis_sock->current = NULL;
@@ -1661,7 +1665,7 @@ redis_mbulk_reply_loop(INTERNAL_FUNCTION_PARAMETERS, RedisSock *redis_sock,
16611665
(unserialize == UNSERIALIZE_VALS && i % 2 != 0)
16621666
);
16631667
zval zv, *z = &zv;
1664-
if (unwrap && redis_unserialize(redis_sock, line, len, z TSRMLS_CC)) {
1668+
if (unwrap && redis_unpack(redis_sock, line, len, z TSRMLS_CC)) {
16651669
#if (PHP_MAJOR_VERSION < 7)
16661670
MAKE_STD_ZVAL(z);
16671671
*z = zv;
@@ -1709,7 +1713,7 @@ PHP_REDIS_API int redis_mbulk_reply_assoc(INTERNAL_FUNCTION_PARAMETERS, RedisSoc
17091713
response = redis_sock_read(redis_sock, &response_len TSRMLS_CC);
17101714
if(response != NULL) {
17111715
zval zv0, *z = &zv0;
1712-
if (redis_unserialize(redis_sock, response, response_len, z TSRMLS_CC)) {
1716+
if (redis_unpack(redis_sock, response, response_len, z TSRMLS_CC)) {
17131717
#if (PHP_MAJOR_VERSION < 7)
17141718
MAKE_STD_ZVAL(z);
17151719
*z = zv0;
@@ -1778,6 +1782,63 @@ PHP_REDIS_API void redis_free_socket(RedisSock *redis_sock)
17781782
efree(redis_sock);
17791783
}
17801784

1785+
PHP_REDIS_API int
1786+
redis_pack(RedisSock *redis_sock, zval *z, char **val, strlen_t *val_len TSRMLS_DC)
1787+
{
1788+
char *buf, *data;
1789+
int valfree;
1790+
strlen_t len;
1791+
uint32_t res;
1792+
1793+
valfree = redis_serialize(redis_sock, z, &buf, &len TSRMLS_CC);
1794+
switch (redis_sock->compression) {
1795+
case REDIS_COMPRESSION_LZF:
1796+
#ifdef HAVE_REDIS_LZF
1797+
data = emalloc(len);
1798+
res = lzf_compress(buf, len, data, len - 1);
1799+
if (res > 0 && res < len) {
1800+
if (valfree) efree(buf);
1801+
*val = data;
1802+
*val_len = res;
1803+
return 1;
1804+
}
1805+
efree(data);
1806+
#endif
1807+
break;
1808+
}
1809+
*val = buf;
1810+
*val_len = len;
1811+
return valfree;
1812+
}
1813+
1814+
PHP_REDIS_API int
1815+
redis_unpack(RedisSock *redis_sock, const char *val, int val_len, zval *z_ret TSRMLS_DC)
1816+
{
1817+
char *data;
1818+
int i, ret;
1819+
uint32_t res;
1820+
1821+
switch (redis_sock->compression) {
1822+
case REDIS_COMPRESSION_LZF:
1823+
#ifdef HAVE_REDIS_LZF
1824+
errno = E2BIG;
1825+
for (i = 1; errno == E2BIG; ++i) {
1826+
data = emalloc(i * val_len);
1827+
if ((res = lzf_decompress(val, val_len, data, i * val_len)) == 0) {
1828+
efree(data);
1829+
continue;
1830+
} else if (redis_unserialize(redis_sock, data, res, z_ret TSRMLS_CC) == 0) {
1831+
ZVAL_STRINGL(z_ret, data, res);
1832+
}
1833+
efree(data);
1834+
return 1;
1835+
}
1836+
#endif
1837+
break;
1838+
}
1839+
return redis_unserialize(redis_sock, val, val_len, z_ret TSRMLS_CC);
1840+
}
1841+
17811842
PHP_REDIS_API int
17821843
redis_serialize(RedisSock *redis_sock, zval *z, char **val, strlen_t *val_len
17831844
TSRMLS_DC)

‎library.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,9 @@ redis_key_prefix(RedisSock *redis_sock, char **key, strlen_t *key_len);
7676
PHP_REDIS_API int
7777
redis_unserialize(RedisSock *redis_sock, const char *val, int val_len, zval *z_ret TSRMLS_DC);
7878

79+
PHP_REDIS_API int redis_pack(RedisSock *redis_sock, zval *z, char **val, strlen_t *val_len TSRMLS_DC);
80+
PHP_REDIS_API int redis_unpack(RedisSock *redis_sock, const char *val, int val_len, zval *z_ret TSRMLS_DC);
81+
7982
/*
8083
* Variant Read methods, mostly to implement eval
8184
*/

‎package.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,9 @@ http://pear.php.net/dtd/package-2.0.xsd">
9292
<file role='src' name='crc16.h'/>
9393
<file role='src' name='redis_session.c'/>
9494
<file role='src' name='redis_session.h'/>
95+
<file role='src' name='liblzf/lzf.h'/>
96+
<file role='src' name='liblzf/lzf_c.c'/>
97+
<file role='src' name='liblzf/lzf_d.c'/>
9598
<dir name='tests'>
9699
<file role='test' name='RedisArrayTest.php'/>
97100
<file role='test' name='RedisClusterTest.php'/>
@@ -117,6 +120,7 @@ http://pear.php.net/dtd/package-2.0.xsd">
117120
<providesextension>redis</providesextension>
118121
<extsrcrelease>
119122
<configureoption name="enable-redis-igbinary" prompt="enable igbinary serializer support?" default="no"/>
123+
<configureoption name="enable-redis-lzf" prompt="enable lzf compression support?" default="no"/>
120124
</extsrcrelease>
121125
<changelog>
122126
<release>

‎redis.c‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -676,6 +676,7 @@ static void add_class_constants(zend_class_entry *ce, int is_cluster TSRMLS_DC)
676676
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_SERIALIZER"), REDIS_OPT_SERIALIZER TSRMLS_CC);
677677
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_PREFIX"), REDIS_OPT_PREFIX TSRMLS_CC);
678678
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_READ_TIMEOUT"), REDIS_OPT_READ_TIMEOUT TSRMLS_CC);
679+
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_COMPRESSION"), REDIS_OPT_COMPRESSION TSRMLS_CC);
679680

680681
/* serializer */
681682
zend_declare_class_constant_long(ce, ZEND_STRL("SERIALIZER_NONE"), REDIS_SERIALIZER_NONE TSRMLS_CC);
@@ -684,6 +685,12 @@ static void add_class_constants(zend_class_entry *ce, int is_cluster TSRMLS_DC)
684685
zend_declare_class_constant_long(ce, ZEND_STRL("SERIALIZER_IGBINARY"), REDIS_SERIALIZER_IGBINARY TSRMLS_CC);
685686
#endif
686687

688+
/* compression */
689+
zend_declare_class_constant_long(ce, ZEND_STRL("COMPRESSION_NONE"), REDIS_COMPRESSION_NONE TSRMLS_CC);
690+
#ifdef HAVE_REDIS_LZF
691+
zend_declare_class_constant_long(ce, ZEND_STRL("COMPRESSION_LZF"), REDIS_COMPRESSION_LZF TSRMLS_CC);
692+
#endif
693+
687694
/* scan options*/
688695
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_SCAN"), REDIS_OPT_SCAN TSRMLS_CC);
689696
zend_declare_class_constant_long(ce, ZEND_STRL("SCAN_RETRY"), REDIS_SCAN_RETRY TSRMLS_CC);
@@ -806,6 +813,9 @@ PHP_MINFO_FUNCTION(redis)
806813
php_info_print_table_row(2, "Available serializers", "php, igbinary");
807814
#else
808815
php_info_print_table_row(2, "Available serializers", "php");
816+
#endif
817+
#ifdef HAVE_REDIS_LZF
818+
php_info_print_table_row(2, "Available compression", "lzf");
809819
#endif
810820
php_info_print_table_end();
811821
}

0 commit comments

Comments
 (0)