Bug fix (CFM tends to HTML encode non-alphanumeric chars in error reports - paths weren't recognized)

stamparm · stamparm · commit 1248fe5eeeed · 2019-02-21T02:50:11.000+01:00
diff --git a/lib/core/settings.py b/lib/core/settings.py
@@ -19,7 +19,7 @@
 from lib.core.enums import OS
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3.2.26"
+VERSION = "1.3.2.27"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -330,7 +330,7 @@
 SESSION_SQLITE_FILE = "session.sqlite"
 
 # Regular expressions used for finding file paths in error messages
-FILE_PATH_REGEXES = (r"<b>(?P<result>[^<>]+?)</b> on line \d+", r"in (?P<result>[^<>'\"]+?)['\"]? on line \d+", r"(?:[>(\[\s])(?P<result>[A-Za-z]:[\\/][\w. \\/-]*)", r"(?:[>(\[\s])(?P<result>/\w[/\w.~-]+)", r"href=['\"]file://(?P<result>/[^'\"]+)")
+FILE_PATH_REGEXES = (r"<b>(?P<result>[^<>]+?)</b> on line \d+", r"\bin (?P<result>[^<>'\"]+?)['\"]? on line \d+", r"(?:[>(\[\s])(?P<result>[A-Za-z]:[\\/][\w. \\/-]*)", r"(?:[>(\[\s])(?P<result>/\w[/\w.~-]+)", r"\bhref=['\"]file://(?P<result>/[^'\"]+)", r"\bin <b>(?P<result>[^<]+): line \d+")
 
 # Regular expressions used for parsing error messages (--parse-errors)
 ERROR_PARSING_REGEXES = (
diff --git a/lib/request/basic.py b/lib/request/basic.py
@@ -313,43 +313,40 @@ def decodePage(page, contentEncoding, contentType):
 
     # can't do for all responses because we need to support binary files too
     if not isinstance(page, unicode) and "text/" in contentType:
-        if kb.heuristicMode:
-            kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page))
-            page = getUnicode(page, kb.pageEncoding)
-        else:
-            # e.g. &#195;&#235;&#224;&#226;&#224;
-            if "&#" in page:
-                page = re.sub(r"&#(\d{1,3});", lambda _: chr(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page)
-
-            # e.g. %20%28%29
-            if "%" in page:
-                page = re.sub(r"%([0-9a-fA-F]{2})", lambda _: _.group(1).decode("hex"), page)
-
-            # e.g. &amp;
-            page = re.sub(r"&([^;]+);", lambda _: chr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 256) < 256 else _.group(0), page)
-
-            kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page))
-
-            if (kb.pageEncoding or "").lower() == "utf-8-sig":
-                kb.pageEncoding = "utf-8"
-                if page and page.startswith("\xef\xbb\xbf"):  # Reference: https://docs.python.org/2/library/codecs.html (Note: noticed problems when "utf-8-sig" is left to Python for handling)
-                    page = page[3:]
-
-            page = getUnicode(page, kb.pageEncoding)
-
-            # e.g. &#8217;&#8230;&#8482;
-            if "&#" in page:
-                def _(match):
-                    retVal = match.group(0)
-                    try:
-                        retVal = unichr(int(match.group(1)))
-                    except (ValueError, OverflowError):
-                        pass
-                    return retVal
-                page = re.sub(r"&#(\d+);", _, page)
-
-            # e.g. &zeta;
-            page = re.sub(r"&([^;]+);", lambda _: unichr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 0) > 255 else _.group(0), page)
+        # e.g. &#x9;&#195;&#235;&#224;&#226;&#224;
+        if "&#" in page:
+            page = re.sub(r"&#x([0-9a-f]{1,2});", lambda _: (_.group(1) if len(_.group(1)) == 2 else "0%s" % _.group(1)).decode("hex"), page)
+            page = re.sub(r"&#(\d{1,3});", lambda _: chr(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page)
+
+        # e.g. %20%28%29
+        if "%" in page:
+            page = re.sub(r"%([0-9a-fA-F]{2})", lambda _: _.group(1).decode("hex"), page)
+
+        # e.g. &amp;
+        page = re.sub(r"&([^;]+);", lambda _: chr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 256) < 256 else _.group(0), page)
+
+        kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page))
+
+        if (kb.pageEncoding or "").lower() == "utf-8-sig":
+            kb.pageEncoding = "utf-8"
+            if page and page.startswith("\xef\xbb\xbf"):  # Reference: https://docs.python.org/2/library/codecs.html (Note: noticed problems when "utf-8-sig" is left to Python for handling)
+                page = page[3:]
+
+        page = getUnicode(page, kb.pageEncoding)
+
+        # e.g. &#8217;&#8230;&#8482;
+        if "&#" in page:
+            def _(match):
+                retVal = match.group(0)
+                try:
+                    retVal = unichr(int(match.group(1)))
+                except (ValueError, OverflowError):
+                    pass
+                return retVal
+            page = re.sub(r"&#(\d+);", _, page)
+
+        # e.g. &zeta;
+        page = re.sub(r"&([^;]+);", lambda _: unichr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 0) > 255 else _.group(0), page)
 
     return page
 
diff --git a/txt/checksum.md5 b/txt/checksum.md5
@@ -50,7 +50,7 @@ d5ef43fe3cdd6c2602d7db45651f9ceb  lib/core/readlineng.py
 7d8a22c582ad201f65b73225e4456170  lib/core/replication.py
 3179d34f371e0295dd4604568fb30bcd  lib/core/revision.py
 d6269c55789f78cf707e09a0f5b45443  lib/core/session.py
-bb7fceee8b646ac156273ecdc2d1d783  lib/core/settings.py
+1ab84830277bc8690adc2e2db916bb8f  lib/core/settings.py
 4483b4a5b601d8f1c4281071dff21ecc  lib/core/shell.py
 10fd19b0716ed261e6d04f311f6f527c  lib/core/subprocessng.py
 43772ea73e9e3d446f782af591cb4eda  lib/core/target.py
@@ -70,7 +70,7 @@ fb6be55d21a70765e35549af2484f762  lib/parse/__init__.py
 adcecd2d6a8667b22872a563eb83eac0  lib/parse/payloads.py
 993104046c7d97120613409ef7780c76  lib/parse/sitemap.py
 e4ea70bcd461f5176867dcd89d372386  lib/request/basicauthhandler.py
-97b7577fdfe3d8537fe9ea3a070d0507  lib/request/basic.py
+b23163d485e0dbc038cbf1ba80be11da  lib/request/basic.py
 fc25d951217077fe655ed2a3a81552ae  lib/request/comparison.py
 2fde12a95133b26699e26a5c56311c38  lib/request/connect.py
 43005bd6a78e9cf0f3ed2283a1cb122e  lib/request/direct.py
