Fix for crawler and redirection case

stamparm · stamparm · commit ebe8ee3500ce · 2013-04-30T18:08:26.000+02:00
diff --git a/lib/core/threads.py b/lib/core/threads.py
@@ -49,6 +49,7 @@ def reset(self):
         self.lastQueryDuration = 0
         self.lastRequestMsg = None
         self.lastRequestUID = 0
+        self.lastRedirectURL = None
         self.resumed = False
         self.retriesCount = 0
         self.seqMatcher = difflib.SequenceMatcher(None)
diff --git a/lib/request/redirecthandler.py b/lib/request/redirecthandler.py
@@ -117,9 +117,10 @@ def http_error_302(self, req, fp, code, msg, headers):
         else:
             result = fp
 
+        threadData.lastRedirectURL = (threadData.lastRequestUID, redurl)
+
         result.redcode = code
         result.redurl = redurl
-
         return result
 
     http_error_301 = http_error_303 = http_error_307 = http_error_302
diff --git a/lib/utils/crawler.py b/lib/utils/crawler.py
@@ -72,6 +72,8 @@ def crawlThread():
                             href = tag.get("href") if hasattr(tag, "get") else tag.group("href")
 
                             if href:
+                                if threadData.lastRedirectURL and threadData.lastRedirectURL[0] == threadData.lastRequestUID:
+                                    current = threadData.lastRedirectURL[1]
                                 url = urlparse.urljoin(current, href)
 
                                 # flag to know if we are dealing with the same target host
