Adding compression to BigArray mechanism (to save disk space on big dumps)

stamparm · stamparm · commit d54ec88648c3 · 2017-11-17T11:45:02.000+01:00
diff --git a/lib/core/bigarray.py b/lib/core/bigarray.py
@@ -14,10 +14,12 @@
 import os
 import sys
 import tempfile
+import zlib
 
 from lib.core.enums import MKSTEMP_PREFIX
 from lib.core.exception import SqlmapSystemException
 from lib.core.settings import BIGARRAY_CHUNK_SIZE
+from lib.core.settings import BIGARRAY_COMPRESS_LEVEL
 
 DEFAULT_SIZE_OF = sys.getsizeof(object())
 
@@ -80,8 +82,8 @@ def pop(self):
         if len(self.chunks[-1]) < 1:
             self.chunks.pop()
             try:
-                with open(self.chunks[-1], "rb") as fp:
-                    self.chunks[-1] = pickle.load(fp)
+                with open(self.chunks[-1], "rb") as f:
+                    self.chunks[-1] = pickle.loads(zlib.decompress(f.read()))
             except IOError, ex:
                 errMsg = "exception occurred while retrieving data "
                 errMsg += "from a temporary file ('%s')" % ex.message
@@ -101,8 +103,8 @@ def _dump(self, chunk):
             handle, filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.BIG_ARRAY)
             self.filenames.add(filename)
             os.close(handle)
-            with open(filename, "w+b") as fp:
-                pickle.dump(chunk, fp, pickle.HIGHEST_PROTOCOL)
+            with open(filename, "w+b") as f:
+                f.write(zlib.compress(pickle.dumps(chunk, pickle.HIGHEST_PROTOCOL), BIGARRAY_COMPRESS_LEVEL))
             return filename
         except (OSError, IOError), ex:
             errMsg = "exception occurred while storing data "
@@ -119,8 +121,8 @@ def _checkcache(self, index):
 
         if not (self.cache and self.cache.index == index):
             try:
-                with open(self.chunks[index], "rb") as fp:
-                    self.cache = Cache(index, pickle.load(fp), False)
+                with open(self.chunks[index], "rb") as f:
+                    self.cache = Cache(index, pickle.loads(zlib.decompress(f.read())), False)
             except IOError, ex:
                 errMsg = "exception occurred while retrieving data "
                 errMsg += "from a temporary file ('%s')" % ex.message
diff --git a/lib/core/settings.py b/lib/core/settings.py
@@ -19,7 +19,7 @@
 from lib.core.enums import OS
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.1.11.21"
+VERSION = "1.1.11.22"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -520,6 +520,9 @@
 # Approximate chunk length (in bytes) used by BigArray objects (only last chunk and cached one are held in memory)
 BIGARRAY_CHUNK_SIZE = 1024 * 1024
 
+# Compress (zlib) level used for storing BigArray chunks to disk (0-9)
+BIGARRAY_COMPRESS_LEVEL = 9
+
 # Maximum number of socket pre-connects
 SOCKET_PRE_CONNECT_QUEUE_SIZE = 3
 
diff --git a/txt/checksum.md5 b/txt/checksum.md5
@@ -26,7 +26,7 @@ f96467fc5cd1d87f52dd7966c8ae6e79  extra/shutils/regressiontest.py
 d2cdb9e832e18a81e936ca3348144b16  lib/controller/handler.py
 5fb9aaf874daa47ea2b672a22740e56b  lib/controller/__init__.py
 fd69e56ce20a5a49ce10a7a745022378  lib/core/agent.py
-d55b4b58019d6dbfddd40ec919f9f172  lib/core/bigarray.py
+8d9d771f7e67582c56a96a8d0ccbe4fc  lib/core/bigarray.py
 16b63aed575a43c1d406df30a9b22afe  lib/core/common.py
 54326d3a690f8b26fe5a5da1a589b369  lib/core/convert.py
 90b1b08368ac8a859300e6fa6a8c796e  lib/core/data.py
@@ -46,7 +46,7 @@ e1c000db9be27f973569b1a430629037  lib/core/option.py
 760d9df2a27ded29109b390ab202e72d  lib/core/replication.py
 a2466b62e67f8b31736bac4dac590e51  lib/core/revision.py
 02d4762140a72fd44668d3dab5eabda9  lib/core/session.py
-915cf4eeb4fefcb6ebb8c10cd3c50468  lib/core/settings.py
+5ca78784a48256f3b8a7bc98b9ec1978  lib/core/settings.py
 35bffbad762eb9e03db9e93b1c991103  lib/core/shell.py
 a59ec28371ae067a6fdd8f810edbee3d  lib/core/subprocessng.py
 d5a04d672a18f78deb2839c3745ff83c  lib/core/target.py
