fix for a bug reported by g@brindi.si (UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2 in position

stamparm · stamparm · commit e9286ddd5b16 · 2011-06-24T19:24:11.000Z
47: ordinal not in range(128))
diff --git a/lib/request/connect.py b/lib/request/connect.py
@@ -133,6 +133,7 @@ def getPage(**kwargs):
         refreshing = kwargs.get('refreshing',       False)
         retrying = kwargs.get('retrying',           False)
         redirecting = kwargs.get('redirecting',     False)
+        crawling = kwargs.get('crawling',           False)
 
         if not urlparse.urlsplit(url).netloc:
             url = urlparse.urljoin(conf.url, url)
@@ -188,7 +189,7 @@ def getPage(**kwargs):
 
                 return page
 
-            elif refreshing:
+            elif any ([refreshing, crawling]):
                 pass
 
             elif target:
diff --git a/lib/utils/crawler.py b/lib/utils/crawler.py
@@ -55,7 +55,7 @@ def crawlThread():
                     content = None
                     try:
                         if current:
-                            content = Request.getPage(url=current, raise404=False)[0]
+                            content = Request.getPage(url=current, crawling=True, raise404=False)[0]
                     except sqlmapConnectionException, e:
                         errMsg = "connection exception detected (%s). skipping " % e
                         errMsg += "url '%s'" % current
