Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 4264900

Browse files
stamparmstamparm
committed
Lots of fixes and refactoring in search department
1 parent b4526a3 commit 4264900

5 files changed

Lines changed: 256 additions & 268 deletions

File tree

‎lib/core/option.py‎

Lines changed: 58 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
import lib.core.threads
2828
import lib.core.convert
2929
import lib.request.connect
30-
import lib.utils.google
30+
import lib.utils.search
3131

3232
from lib.controller.checks import checkConnection
3333
from lib.core.common import Backend
@@ -148,7 +148,7 @@
148148
from lib.request.templates import getPageTemplate
149149
from lib.utils.crawler import crawl
150150
from lib.utils.deps import checkDependencies
151-
from lib.utils.google import Google
151+
from lib.utils.search import search
152152
from lib.utils.purge import purge
153153
from thirdparty.colorama.initialise import init as coloramainit
154154
from thirdparty.keepalive import keepalive
@@ -503,46 +503,23 @@ def _setCrawler():
503503
errMsg = "problem occurred while crawling at '%s' ('%s')" % (target, ex)
504504
logger.error(errMsg)
505505

506-
def _setGoogleDorking():
506+
def _doSearch():
507507
"""
508-
This function checks if the way to request testable hosts is through
509-
Google dorking then requests to Google the search parameter, parses
510-
the results and save the testable hosts into the knowledge base.
508+
This function performs search dorking, parses results
509+
and saves the testable hosts into the knowledge base.
511510
"""
512511

513512
if not conf.googleDork:
514513
return
515514

516-
global keepAliveHandler
517-
global proxyHandler
518-
519-
debugMsg = "initializing Google dorking requests"
520-
logger.debug(debugMsg)
521-
522-
infoMsg = "first request to Google to get the session cookie"
523-
logger.info(infoMsg)
524-
525-
handlers = [proxyHandler]
526-
527-
# Reference: http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html
528-
if conf.keepAlive:
529-
if conf.proxy:
530-
warnMsg = "persistent HTTP(s) connections, Keep-Alive, has "
531-
warnMsg += "been disabled because of its incompatibility "
532-
warnMsg += "with HTTP(s) proxy"
533-
logger.warn(warnMsg)
534-
else:
535-
handlers.append(keepAliveHandler)
536-
537-
googleObj = Google(handlers)
538515
kb.data.onlyGETs = None
539516

540517
def retrieve():
541-
links = googleObj.search(conf.googleDork)
518+
links = search(conf.googleDork)
542519

543520
if not links:
544521
errMsg = "unable to find results for your "
545-
errMsg += "Google dork expression"
522+
errMsg += "search dork expression"
546523
raise SqlmapGenericException(errMsg)
547524

548525
for link in links:
@@ -564,7 +541,7 @@ def retrieve():
564541

565542
if kb.targets:
566543
infoMsg = "sqlmap got %d results for your " % len(links)
567-
infoMsg += "Google dork expression, "
544+
infoMsg += "search dork expression, "
568545

569546
if len(links) == len(kb.targets):
570547
infoMsg += "all "
@@ -577,7 +554,7 @@ def retrieve():
577554

578555
else:
579556
message = "sqlmap got %d results " % len(links)
580-
message += "for your Google dork expression, but none of them "
557+
message += "for your search dork expression, but none of them "
581558
message += "have GET parameters to test for SQL injection. "
582559
message += "Do you want to skip to the next result page? [Y/n]"
583560
test = readInput(message, default="Y")
@@ -1041,7 +1018,7 @@ def _getaddrinfo(*args, **kwargs):
10411018
socket._getaddrinfo = socket.getaddrinfo
10421019
socket.getaddrinfo = _getaddrinfo
10431020

1044-
def _setHTTPProxy():
1021+
def _setHTTPHandlers():
10451022
"""
10461023
Check and set the HTTP/SOCKS proxy for all HTTP requests.
10471024
"""
@@ -1066,63 +1043,62 @@ def _setHTTPProxy():
10661043
if conf.hostname in ("localhost", "127.0.0.1") or conf.ignoreProxy:
10671044
proxyHandler.proxies = {}
10681045

1069-
return
1070-
1071-
debugMsg = "setting the HTTP/SOCKS proxy for all HTTP requests"
1072-
logger.debug(debugMsg)
1073-
1074-
try:
1075-
_ = urlparse.urlsplit(conf.proxy)
1076-
except Exception, ex:
1077-
errMsg = "invalid proxy address '%s' ('%s')" % (conf.proxy, ex)
1078-
raise SqlmapSyntaxException, errMsg
1046+
if conf.proxy:
1047+
debugMsg = "setting the HTTP/SOCKS proxy for all HTTP requests"
1048+
logger.debug(debugMsg)
10791049

1080-
hostnamePort = _.netloc.split(":")
1050+
try:
1051+
_ = urlparse.urlsplit(conf.proxy)
1052+
except Exception, ex:
1053+
errMsg = "invalid proxy address '%s' ('%s')" % (conf.proxy, ex)
1054+
raise SqlmapSyntaxException, errMsg
10811055

1082-
scheme = _.scheme.upper()
1083-
hostname = hostnamePort[0]
1084-
port = None
1085-
username = None
1086-
password = None
1056+
hostnamePort = _.netloc.split(":")
10871057

1088-
if len(hostnamePort) == 2:
1089-
try:
1090-
port = int(hostnamePort[1])
1091-
except:
1092-
pass # drops into the next check block
1058+
scheme = _.scheme.upper()
1059+
hostname = hostnamePort[0]
1060+
port = None
1061+
username = None
1062+
password = None
10931063

1094-
if not all((scheme, hasattr(PROXY_TYPE, scheme), hostname, port)):
1095-
errMsg = "proxy value must be in format '(%s)://address:port'" % "|".join(_[0].lower() for _ in getPublicTypeMembers(PROXY_TYPE))
1096-
raise SqlmapSyntaxException(errMsg)
1064+
if len(hostnamePort) == 2:
1065+
try:
1066+
port = int(hostnamePort[1])
1067+
except:
1068+
pass # drops into the next check block
10971069

1098-
if conf.proxyCred:
1099-
_ = re.search("^(.*?):(.*?)$", conf.proxyCred)
1100-
if not _:
1101-
errMsg = "proxy authentication credentials "
1102-
errMsg += "value must be in format username:password"
1070+
if not all((scheme, hasattr(PROXY_TYPE, scheme), hostname, port)):
1071+
errMsg = "proxy value must be in format '(%s)://address:port'" % "|".join(_[0].lower() for _ in getPublicTypeMembers(PROXY_TYPE))
11031072
raise SqlmapSyntaxException(errMsg)
1104-
else:
1105-
username = _.group(1)
1106-
password = _.group(2)
11071073

1108-
if scheme in (PROXY_TYPE.SOCKS4, PROXY_TYPE.SOCKS5):
1109-
proxyHandler.proxies = {}
1074+
if conf.proxyCred:
1075+
_ = re.search("^(.*?):(.*?)$", conf.proxyCred)
1076+
if not _:
1077+
errMsg = "proxy authentication credentials "
1078+
errMsg += "value must be in format username:password"
1079+
raise SqlmapSyntaxException(errMsg)
1080+
else:
1081+
username = _.group(1)
1082+
password = _.group(2)
11101083

1111-
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5 if scheme == PROXY_TYPE.SOCKS5 else socks.PROXY_TYPE_SOCKS4, hostname, port, username=username, password=password)
1112-
socks.wrapmodule(urllib2)
1113-
else:
1114-
socks.unwrapmodule(urllib2)
1084+
if scheme in (PROXY_TYPE.SOCKS4, PROXY_TYPE.SOCKS5):
1085+
proxyHandler.proxies = {}
11151086

1116-
if conf.proxyCred:
1117-
# Reference: http://stackoverflow.com/questions/34079/how-to-specify-an-authenticated-proxy-for-a-python-http-connection
1118-
proxyString = "%s@" % conf.proxyCred
1087+
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5 if scheme == PROXY_TYPE.SOCKS5 else socks.PROXY_TYPE_SOCKS4, hostname, port, username=username, password=password)
1088+
socks.wrapmodule(urllib2)
11191089
else:
1120-
proxyString = ""
1090+
socks.unwrapmodule(urllib2)
1091+
1092+
if conf.proxyCred:
1093+
# Reference: http://stackoverflow.com/questions/34079/how-to-specify-an-authenticated-proxy-for-a-python-http-connection
1094+
proxyString = "%s@" % conf.proxyCred
1095+
else:
1096+
proxyString = ""
11211097

1122-
proxyString += "%s:%d" % (hostname, port)
1123-
proxyHandler.proxies = {"http": proxyString, "https": proxyString}
1098+
proxyString += "%s:%d" % (hostname, port)
1099+
proxyHandler.proxies = {"http": proxyString, "https": proxyString}
11241100

1125-
proxyHandler.__init__(proxyHandler.proxies)
1101+
proxyHandler.__init__(proxyHandler.proxies)
11261102

11271103
debugMsg = "creating HTTP requests opener object"
11281104
logger.debug(debugMsg)
@@ -2489,8 +2465,8 @@ def _resolveCrossReferences():
24892465
lib.core.threads.readInput = readInput
24902466
lib.core.common.getPageTemplate = getPageTemplate
24912467
lib.core.convert.singleTimeWarnMessage = singleTimeWarnMessage
2492-
lib.request.connect.setHTTPProxy = _setHTTPProxy
2493-
lib.utils.google.setHTTPProxy = _setHTTPProxy
2468+
lib.request.connect.setHTTPHandlers = _setHTTPHandlers
2469+
lib.utils.search.setHTTPHandlers = _setHTTPHandlers
24942470
lib.controller.checks.setVerbosity = setVerbosity
24952471

24962472
def initOptions(inputOptions=AttribDict(), overrideOptions=False):
@@ -2539,10 +2515,10 @@ def init():
25392515
_setHTTPHost()
25402516
_setHTTPUserAgent()
25412517
_setHTTPAuthentication()
2542-
_setHTTPProxy()
2518+
_setHTTPHandlers()
25432519
_setDNSCache()
25442520
_setSafeVisit()
2545-
_setGoogleDorking()
2521+
_doSearch()
25462522
_setBulkMultipleTargets()
25472523
_setSitemapTargets()
25482524
_checkTor()

‎lib/core/settings.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@
7373
MAX_CONNECTIONS_REGEX = r"max.+connections"
7474

7575
# Regular expression used for extracting results from Google search
76-
GOOGLE_REGEX = r"url\?\w+=((?![^>]+webcache\.googleusercontent\.com)http[^>]+)&(sa=U|rct=j)"
76+
GOOGLE_REGEX = r"webcache\.googleusercontent\.com/search\?q=cache:[^:]+:([^+]+)\+&cd=|url\?\w+=((?![^>]+webcache\.googleusercontent\.com)http[^>]+)&(sa=U|rct=j)"
7777

7878
# Regular expression used for extracting results from DuckDuckGo search
7979
DUCKDUCKGO_REGEX = r'"u":"([^"]+)'

‎lib/request/connect.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ def _retryProxy(**kwargs):
134134
conf.proxy = None
135135
threadData.retriesCount = 0
136136

137-
setHTTPProxy()
137+
setHTTPHandlers()
138138

139139
if kb.testMode and kb.previousMethod == PAYLOAD.METHOD.TIME:
140140
# timed based payloads can cause web server unresponsiveness
@@ -1118,5 +1118,5 @@ def _randomizeParameter(paramString, randomParameter):
11181118
else:
11191119
return comparison(page, headers, code, getRatioValue, pageLength)
11201120

1121-
def setHTTPProxy(): # Cross-linked function
1121+
def setHTTPHandlers(): # Cross-linked function
11221122
raise NotImplementedError

0 commit comments

Comments
 (0)