Minor update for crawler

stamparm · stamparm · commit 2bfdac5ebcdd · 2013-04-30T18:32:46.000+02:00
diff --git a/lib/utils/crawler.py b/lib/utils/crawler.py
@@ -27,6 +27,7 @@
 
 def crawl(target):
     try:
+        visited = set()
         threadData = getCurrentThreadData()
         threadData.shared.value = oset()
 
@@ -37,6 +38,10 @@ def crawlThread():
                 with kb.locks.limit:
                     if threadData.shared.unprocessed:
                         current = threadData.shared.unprocessed.pop()
+                        if current in visited:
+                            continue
+                        else:
+                            visited.add(current)
                     else:
                         break
 
