Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 961d2b2

Browse files
stamparmstamparm
committed
First commit regarding #739 
1 parent 53578bc commit 961d2b2

4 files changed

Lines changed: 217 additions & 210 deletions

File tree

‎lib/core/common.py‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
See the file 'LICENSE' for copying permission
66
"""
77

8+
import binascii
89
import codecs
910
import contextlib
1011
import cookielib
@@ -101,7 +102,10 @@
101102
from lib.core.settings import BRUTE_DOC_ROOT_PREFIXES
102103
from lib.core.settings import BRUTE_DOC_ROOT_SUFFIXES
103104
from lib.core.settings import BRUTE_DOC_ROOT_TARGET_MARK
105+
from lib.core.settings import BURP_REQUEST_REGEX
106+
from lib.core.settings import BURP_XML_HISTORY_REGEX
104107
from lib.core.settings import DBMS_DIRECTORY_DICT
108+
from lib.core.settings import CRAWL_EXCLUDE_EXTENSIONS
105109
from lib.core.settings import CUSTOM_INJECTION_MARK_CHAR
106110
from lib.core.settings import DEFAULT_COOKIE_DELIMITER
107111
from lib.core.settings import DEFAULT_GET_POST_DELIMITER
@@ -139,6 +143,7 @@
139143
from lib.core.settings import PAYLOAD_DELIMITER
140144
from lib.core.settings import PLATFORM
141145
from lib.core.settings import PRINTABLE_CHAR_REGEX
146+
from lib.core.settings import PROBLEMATIC_CUSTOM_INJECTION_PATTERNS
142147
from lib.core.settings import PUSH_VALUE_EXCEPTION_RETRY_COUNT
143148
from lib.core.settings import PYVERSION
144149
from lib.core.settings import REFERER_ALIASES
@@ -161,6 +166,7 @@
161166
from lib.core.settings import URLENCODE_FAILSAFE_CHARS
162167
from lib.core.settings import USER_AGENT_ALIASES
163168
from lib.core.settings import VERSION_STRING
169+
from lib.core.settings import WEBSCARAB_SPLITTER
164170
from lib.core.threads import getCurrentThreadData
165171
from lib.utils.sqlalchemy import _sqlalchemy
166172
from thirdparty.clientform.clientform import ParseResponse
@@ -4468,6 +4474,195 @@ def pollProcess(process, suppress_errors=False):
44684474

44694475
break
44704476

4477+
def parseRequestFile(reqFile):
4478+
"""
4479+
Parses WebScarab and Burp logs and adds results to the target URL list
4480+
"""
4481+
4482+
def _parseWebScarabLog(content):
4483+
"""
4484+
Parses WebScarab logs (POST method not supported)
4485+
"""
4486+
4487+
reqResList = content.split(WEBSCARAB_SPLITTER)
4488+
4489+
for request in reqResList:
4490+
url = extractRegexResult(r"URL: (?P<result>.+?)\n", request, re.I)
4491+
method = extractRegexResult(r"METHOD: (?P<result>.+?)\n", request, re.I)
4492+
cookie = extractRegexResult(r"COOKIE: (?P<result>.+?)\n", request, re.I)
4493+
4494+
if not method or not url:
4495+
logger.debug("not a valid WebScarab log data")
4496+
continue
4497+
4498+
if method.upper() == HTTPMETHOD.POST:
4499+
warnMsg = "POST requests from WebScarab logs aren't supported "
4500+
warnMsg += "as their body content is stored in separate files. "
4501+
warnMsg += "Nevertheless you can use -r to load them individually."
4502+
logger.warning(warnMsg)
4503+
continue
4504+
4505+
if not(conf.scope and not re.search(conf.scope, url, re.I)):
4506+
yield (url, method, None, cookie, None)
4507+
4508+
def _parseBurpLog(content):
4509+
"""
4510+
Parses Burp logs
4511+
"""
4512+
4513+
if not re.search(BURP_REQUEST_REGEX, content, re.I | re.S):
4514+
if re.search(BURP_XML_HISTORY_REGEX, content, re.I | re.S):
4515+
reqResList = []
4516+
for match in re.finditer(BURP_XML_HISTORY_REGEX, content, re.I | re.S):
4517+
port, request = match.groups()
4518+
try:
4519+
request = request.decode("base64")
4520+
except binascii.Error:
4521+
continue
4522+
_ = re.search(r"%s:.+" % re.escape(HTTP_HEADER.HOST), request)
4523+
if _:
4524+
host = _.group(0).strip()
4525+
if not re.search(r":\d+\Z", host):
4526+
request = request.replace(host, "%s:%d" % (host, int(port)))
4527+
reqResList.append(request)
4528+
else:
4529+
reqResList = [content]
4530+
else:
4531+
reqResList = re.finditer(BURP_REQUEST_REGEX, content, re.I | re.S)
4532+
4533+
for match in reqResList:
4534+
request = match if isinstance(match, basestring) else match.group(0)
4535+
request = re.sub(r"\A[^\w]+", "", request)
4536+
4537+
schemePort = re.search(r"(http[\w]*)\:\/\/.*?\:([\d]+).+?={10,}", request, re.I | re.S)
4538+
4539+
if schemePort:
4540+
scheme = schemePort.group(1)
4541+
port = schemePort.group(2)
4542+
request = re.sub(r"\n=+\Z", "", request.split(schemePort.group(0))[-1].lstrip())
4543+
else:
4544+
scheme, port = None, None
4545+
4546+
if not re.search(r"^[\n]*(%s).*?\sHTTP\/" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), request, re.I | re.M):
4547+
continue
4548+
4549+
if re.search(r"^[\n]*%s.*?\.(%s)\sHTTP\/" % (HTTPMETHOD.GET, "|".join(CRAWL_EXCLUDE_EXTENSIONS)), request, re.I | re.M):
4550+
continue
4551+
4552+
getPostReq = False
4553+
url = None
4554+
host = None
4555+
method = None
4556+
data = None
4557+
cookie = None
4558+
params = False
4559+
newline = None
4560+
lines = request.split('\n')
4561+
headers = []
4562+
4563+
for index in xrange(len(lines)):
4564+
line = lines[index]
4565+
4566+
if not line.strip() and index == len(lines) - 1:
4567+
break
4568+
4569+
newline = "\r\n" if line.endswith('\r') else '\n'
4570+
line = line.strip('\r')
4571+
match = re.search(r"\A(%s) (.+) HTTP/[\d.]+\Z" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), line) if not method else None
4572+
4573+
if len(line.strip()) == 0 and method and method != HTTPMETHOD.GET and data is None:
4574+
data = ""
4575+
params = True
4576+
4577+
elif match:
4578+
method = match.group(1)
4579+
url = match.group(2)
4580+
4581+
if any(_ in line for _ in ('?', '=', kb.customInjectionMark)):
4582+
params = True
4583+
4584+
getPostReq = True
4585+
4586+
# POST parameters
4587+
elif data is not None and params:
4588+
data += "%s%s" % (line, newline)
4589+
4590+
# GET parameters
4591+
elif "?" in line and "=" in line and ": " not in line:
4592+
params = True
4593+
4594+
# Headers
4595+
elif re.search(r"\A\S+:", line):
4596+
key, value = line.split(":", 1)
4597+
value = value.strip().replace("\r", "").replace("\n", "")
4598+
4599+
# Cookie and Host headers
4600+
if key.upper() == HTTP_HEADER.COOKIE.upper():
4601+
cookie = value
4602+
elif key.upper() == HTTP_HEADER.HOST.upper():
4603+
if '://' in value:
4604+
scheme, value = value.split('://')[:2]
4605+
splitValue = value.split(":")
4606+
host = splitValue[0]
4607+
4608+
if len(splitValue) > 1:
4609+
port = filterStringValue(splitValue[1], "[0-9]")
4610+
4611+
# Avoid to add a static content length header to
4612+
# headers and consider the following lines as
4613+
# POSTed data
4614+
if key.upper() == HTTP_HEADER.CONTENT_LENGTH.upper():
4615+
params = True
4616+
4617+
# Avoid proxy and connection type related headers
4618+
elif key not in (HTTP_HEADER.PROXY_CONNECTION, HTTP_HEADER.CONNECTION):
4619+
headers.append((getUnicode(key), getUnicode(value)))
4620+
4621+
if kb.customInjectionMark in re.sub(PROBLEMATIC_CUSTOM_INJECTION_PATTERNS, "", value or ""):
4622+
params = True
4623+
4624+
data = data.rstrip("\r\n") if data else data
4625+
4626+
if getPostReq and (params or cookie):
4627+
if not port and isinstance(scheme, basestring) and scheme.lower() == "https":
4628+
port = "443"
4629+
elif not scheme and port == "443":
4630+
scheme = "https"
4631+
4632+
if conf.forceSSL:
4633+
scheme = "https"
4634+
port = port or "443"
4635+
4636+
if not host:
4637+
errMsg = "invalid format of a request file"
4638+
raise SqlmapSyntaxException(errMsg)
4639+
4640+
if not url.startswith("http"):
4641+
url = "%s://%s:%s%s" % (scheme or "http", host, port or "80", url)
4642+
scheme = None
4643+
port = None
4644+
4645+
if not(conf.scope and not re.search(conf.scope, url, re.I)):
4646+
yield (url, conf.method or method, data, cookie, tuple(headers))
4647+
4648+
checkFile(reqFile)
4649+
try:
4650+
with openFile(reqFile, "rb") as f:
4651+
content = f.read()
4652+
except (IOError, OSError, MemoryError), ex:
4653+
errMsg = "something went wrong while trying "
4654+
errMsg += "to read the content of file '%s' ('%s')" % (reqFile, getSafeExString(ex))
4655+
raise SqlmapSystemException(errMsg)
4656+
4657+
if conf.scope:
4658+
logger.info("using regular expression '%s' for filtering targets" % conf.scope)
4659+
4660+
for target in _parseBurpLog(content):
4661+
yield target
4662+
4663+
for target in _parseWebScarabLog(content):
4664+
yield target
4665+
44714666
def getSafeExString(ex, encoding=None):
44724667
"""
44734668
Safe way how to get the proper exception represtation as a string

0 commit comments

Comments
 (0)