reading page responses in chunks, trimming unnecessary content (especially for large table dumps in full inband cases)

stamparm · stamparm · commit b2afa87e487a · 2012-04-06T08:42:36.000Z
diff --git a/lib/core/settings.py b/lib/core/settings.py
@@ -475,3 +475,9 @@
 
 # Reference: http://www.tcpipguide.com/free/t_DNSLabelsNamesandSyntaxRules.htm
 MAX_DNS_LABEL = 63
+
+# Connection chunk size (processing large responses in chunks to avoid MemoryError crashes - e.g. large table dump in full UNION/inband injections)
+MAX_CONNECTION_CHUNK_SIZE = 10 * 1024 * 1024
+
+# Mark used for trimming unnecessary content in large chunks
+LARGE_CHUNK_TRIM_MARKER = "__TRIMMED_CONTENT__"
diff --git a/lib/request/connect.py b/lib/request/connect.py
@@ -52,12 +52,14 @@
 from lib.core.exception import sqlmapSyntaxException
 from lib.core.settings import HTTP_ACCEPT_HEADER_VALUE
 from lib.core.settings import HTTP_SILENT_TIMEOUT
+from lib.core.settings import MAX_CONNECTION_CHUNK_SIZE
 from lib.core.settings import META_REFRESH_REGEX
-from lib.core.settings import IS_WIN
 from lib.core.settings import MIN_TIME_RESPONSES
-from lib.core.settings import WARN_TIME_STDEV
+from lib.core.settings import IS_WIN
+from lib.core.settings import LARGE_CHUNK_TRIM_MARKER
 from lib.core.settings import UNENCODED_ORIGINAL_VALUE
 from lib.core.settings import URI_HTTP_HEADER
+from lib.core.settings import WARN_TIME_STDEV
 from lib.request.basic import decodePage
 from lib.request.basic import forgeHeaders
 from lib.request.basic import processResponse
@@ -117,6 +119,21 @@ def __retryProxy(**kwargs):
         kwargs['retrying'] = True
         return Connect.__getPageProxy(**kwargs)
 
+    @staticmethod
+    def __connReadProxy(conn):
+        retVal = ""
+        while True:
+            _ = conn.read(MAX_CONNECTION_CHUNK_SIZE)
+            if len(_) == MAX_CONNECTION_CHUNK_SIZE:
+                warnMsg = "large response detected. This could take a while"
+                singleTimeWarnMessage(warnMsg)
+                _ = re.sub(r"(?si)%s.+?%s" % (kb.chars.stop, kb.chars.start), "%s%s%s" % (kb.chars.stop, LARGE_CHUNK_TRIM_MARKER, kb.chars.start), _)
+                retVal += _
+            else:
+                retVal += _
+                break
+        return retVal
+
     @staticmethod
     def getPage(**kwargs):
         """
@@ -205,7 +222,7 @@ def getPage(**kwargs):
 
                 multipartOpener = urllib2.build_opener(proxyHandler, multipartpost.MultipartPostHandler)
                 conn = multipartOpener.open(unicodeencode(url), multipart)
-                page = conn.read()
+                page = Connect.__connReadProxy(conn)
                 responseHeaders = conn.info()
                 responseHeaders[URI_HTTP_HEADER] = conn.geturl()
                 page = decodePage(page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE))
@@ -306,11 +323,11 @@ def getPage(**kwargs):
             # Get HTTP response
             if hasattr(conn, 'redurl'):
                 page = threadData.lastRedirectMsg[1] if kb.redirectChoice == REDIRECTION.NO\
-                  else conn.read()
+                  else Connect.__connReadProxy(conn)
                 skipLogTraffic = kb.redirectChoice == REDIRECTION.NO
                 code = conn.redcode
             else:
-                page = conn.read()
+                page = Connect.__connReadProxy(conn)
 
             code = code or conn.code
             responseHeaders = conn.info()
