Update for an Issue #565 (more work TBD - DuckDuckGo has some kind of IP blocking mechanism)

stamparm · stamparm · commit 7054586e8ac1 · 2013-11-25T20:57:07.000+01:00
diff --git a/lib/core/settings.py b/lib/core/settings.py
@@ -53,9 +53,12 @@
 # Regular expression used for recognition of generic maximum connection messages
 MAX_CONNECTIONS_REGEX = r"max.+connections"
 
-# Regular expression used for extracting results from google search
+# Regular expression used for extracting results from Google search
 GOOGLE_REGEX = r"url\?\w+=((?![^>]+webcache\.googleusercontent\.com)http[^>]+)&(sa=U|rct=j)"
 
+# Regular expression used for extracting results from DuckDuckGo search
+DUCKDUCKGO_REGEX = r'"u":"([^"]+)'
+
 # Regular expression used for extracting content from "textual" tags
 TEXT_TAG_REGEX = r"(?si)<(abbr|acronym|b|blockquote|br|center|cite|code|dt|em|font|h\d|i|li|p|pre|q|strong|sub|sup|td|th|title|tt|u)(?!\w).*?>(?P<result>[^<]+)"
 
diff --git a/lib/utils/google.py b/lib/utils/google.py
@@ -13,13 +13,16 @@
 import urllib2
 
 from lib.core.common import getUnicode
+from lib.core.common import readInput
 from lib.core.common import urlencode
 from lib.core.data import conf
 from lib.core.data import logger
 from lib.core.enums import CUSTOM_LOGGING
+from lib.core.enums import HTTP_HEADER
 from lib.core.exception import SqlmapConnectionException
 from lib.core.exception import SqlmapGenericException
 from lib.core.settings import GOOGLE_REGEX
+from lib.core.settings import DUCKDUCKGO_REGEX
 from lib.core.settings import UNICODE_ENCODING
 from lib.request.basic import decodePage
 
@@ -103,4 +106,52 @@ def search(self, dork):
             warnMsg += "used IP address disabling further searches"
             raise SqlmapGenericException(warnMsg)
 
+        if not retVal:
+            message = "no usable links found. "
+            message += "do you want to (re)try with DuckDuckGo? [Y/n] "
+            output = readInput(message, default="Y")
+
+            if output.strip().lower() != 'n':
+                url = "https://duckduckgo.com/d.js?"
+                url += "q=%s&p=%d&s=100" % (urlencode(dork, convall=True), gpage)
+
+                if not conf.randomAgent:
+                    conf.opener.addheaders = [_ for _ in conf.opener.addheaders if _[0].lower() != HTTP_HEADER.USER_AGENT.lower()]
+                    conf.opener.addheaders.append((HTTP_HEADER.USER_AGENT, "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0"))
+
+                try:
+                    conn = self.opener.open(url)
+
+                    requestMsg = "HTTP request:\nGET %s" % url
+                    requestMsg += " %s" % httplib.HTTPConnection._http_vsn_str
+                    logger.log(CUSTOM_LOGGING.TRAFFIC_OUT, requestMsg)
+
+                    page = conn.read()
+                    code = conn.code
+                    status = conn.msg
+                    responseHeaders = conn.info()
+                    page = decodePage(page, responseHeaders.get("Content-Encoding"), responseHeaders.get("Content-Type"))
+
+                    responseMsg = "HTTP response (%s - %d):\n" % (status, code)
+
+                    if conf.verbose <= 4:
+                        responseMsg += getUnicode(responseHeaders, UNICODE_ENCODING)
+                    elif conf.verbose > 4:
+                        responseMsg += "%s\n%s\n" % (responseHeaders, page)
+
+                    logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg)
+                except urllib2.HTTPError, e:
+                    try:
+                        page = e.read()
+                    except socket.timeout:
+                        warnMsg = "connection timed out while trying "
+                        warnMsg += "to get error page information (%d)" % e.code
+                        logger.critical(warnMsg)
+                        return None
+                except (urllib2.URLError, socket.error, socket.timeout):
+                    errMsg = "unable to connect to DuckDuckGo"
+                    raise SqlmapConnectionException(errMsg)
+
+            retVal = [urllib.unquote(match.group(1)) for match in re.finditer(DUCKDUCKGO_REGEX, page, re.I | re.S)]
+
         return retVal
